<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2008-2013, Bryan Davis
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#     - Redistributions of source code must retain the above copyright notice,
#     this list of conditions and the following disclaimer.
#     - Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

__all__ = (
    'cidr2block',
    'ip2long',
    'long2ip',
    'validate_cidr',
    'validate_ip',
    'MAX_IP',
    'MIN_IP',
)

import re
from . import ipv4

#: Regex for validating an IPv6 in hex notation
_HEX_RE = re.compile(r'^([0-9a-f]{0,4}:){2,7}[0-9a-f]{0,4}$')

#: Regex for validating an IPv6 in dotted-quad notation
_DOTTED_QUAD_RE = re.compile(r'^([0-9a-f]{0,4}:){2,6}(\d{1,3}\.){0,3}\d{1,3}$')

#: Regex for validating a CIDR network
_CIDR_RE = re.compile(r'^([0-9a-f]{0,4}:){2,7}[0-9a-f]{0,4}/\d{1,3}$')

#: Mamimum IPv6 integer
MAX_IP = 0xffffffffffffffffffffffffffffffff
#: Minimum IPv6 integer
MIN_IP = 0x0

def validate_ip(s):
    """Validate a hexidecimal IPv6 ip address.

    >>> validate_ip('::')
    True
    >>> validate_ip('::1')
    True
    >>> validate_ip('2001:db8:85a3::8a2e:370:7334')
    True
    >>> validate_ip('2001:db8:85a3:0:0:8a2e:370:7334')
    True
    >>> validate_ip('2001:0db8:85a3:0000:0000:8a2e:0370:7334')
    True
    >>> validate_ip('2001:db8::1:0:0:1')
    True
    >>> validate_ip('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff')
    True
    >>> validate_ip('::ffff:192.0.2.128')
    True
    >>> validate_ip('::ff::ff')
    False
    >>> validate_ip('::fffff')
    False
    >>> validate_ip('::ffff:192.0.2.300')
    False
    >>> validate_ip(None) #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: expected string or buffer

    :param s: String to validate as a hexidecimal IPv6 ip address.
    :type s: str
    :returns: ``True`` if a valid hexidecimal IPv6 ip address,
              ``False`` otherwise.
    :raises: TypeError
    """
    if _HEX_RE.match(s):
        return len(s.split('::')) <= 2
    if _DOTTED_QUAD_RE.match(s):
        halves = s.split('::')
        if len(halves) > 2:
            return False
        hextets = s.split(':')
        quads = hextets[-1].split('.')
        for q in quads:
            if int(q) > 255:
                return False
        return True
    return False
#end validate_ip

def ip2long(ip):
    """Convert a hexidecimal IPv6 address to a network byte order 128-bit
    integer.

    >>> ip2long('::') == 0
    True
    >>> ip2long('::1') == 1
    True
    >>> expect = 0x20010db885a3000000008a2e03707334
    >>> ip2long('2001:db8:85a3::8a2e:370:7334') == expect
    True
    >>> ip2long('2001:db8:85a3:0:0:8a2e:370:7334') == expect
    True
    >>> ip2long('2001:0db8:85a3:0000:0000:8a2e:0370:7334') == expect
    True
    >>> expect = 0x20010db8000000000001000000000001
    >>> ip2long('2001:db8::1:0:0:1') == expect
    True
    >>> expect = 281473902969472
    >>> ip2long('::ffff:192.0.2.128') == expect
    True
    >>> expect = 0xffffffffffffffffffffffffffffffff
    >>> ip2long('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff') == expect
    True
    >>> ip2long('ff::ff::ff') == None
    True

    :param ip: Hexidecimal IPv6 address
    :type ip: str
    :returns: Network byte order 128-bit integer or ``None`` if ip is invalid.
    """
    if not validate_ip(ip):
        return None

    if '.' in ip:
        # convert IPv4 suffix to hex
        chunks = ip.split(':')
        v4_int = ipv4.ip2long(chunks.pop())
        if v4_int is None:
            return None
        chunks.append('%x' % ((v4_int >> 16) & 0xffff))
        chunks.append('%x' % (v4_int & 0xffff))
        ip = ':'.join(chunks)

    halves = ip.split('::')
    hextets = halves[0].split(':')
    if len(halves) == 2:
        h2 = halves[1].split(':')
        for z in xrange(8 - (len(hextets) + len(h2))):
            hextets.append('0')
        for h in h2:
            hextets.append(h)
    #end if

    lngip = 0
    for h in hextets:
        if '' == h:
            h = '0'
        lngip = (lngip << 16) | int(h, 16)
    return lngip
#end ip2long

def long2ip(l):
    """Convert a network byte order 128-bit integer to a canonical IPv6
    address.

    >>> long2ip(2130706433)
    '::7f00:1'
    >>> long2ip(42540766411282592856904266426630537217)
    '2001:db8::1:0:0:1'
    >>> long2ip(MIN_IP)
    '::'
    >>> long2ip(MAX_IP)
    'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff'
    >>> long2ip(None) #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: unsupported operand type(s) for >>: 'NoneType' and 'int'
    >>> long2ip(-1) #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: expected int between 0 and <really big int> inclusive
    >>> long2ip(MAX_IP + 1) #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: expected int between 0 and <really big int> inclusive

    :param l: Network byte order 128-bit integer.
    :type l: int
    :returns: Canonical IPv6 address (eg. '::1').
    :raises: TypeError
    """
    if MAX_IP < l or l < MIN_IP:
        raise TypeError(
            "expected int between %d and %d inclusive" % (MIN_IP, MAX_IP))

    # format as one big hex value
    hex_str = '%032x' % l
    # split into double octet chunks without padding zeros
    hextets = ['%x' % int(hex_str[x:x+4], 16) for x in range(0, 32, 4)]

    # find and remove left most longest run of zeros
    dc_start, dc_len = (-1, 0)
    run_start, run_len = (-1, 0)
    for idx, hextet in enumerate(hextets):
        if '0' == hextet:
            run_len += 1
            if -1 == run_start:
                run_start = idx
            if run_len > dc_len:
                dc_len, dc_start = (run_len, run_start)
        else:
            run_len, run_start = (0, -1)
    #end for
    if dc_len > 1:
        dc_end = dc_start + dc_len
        if dc_end == len(hextets):
            hextets += ['']
        hextets[dc_start:dc_end] = ['']
        if dc_start == 0:
            hextets = [''] + hextets
    #end if

    return ':'.join(hextets)
#end long2ip

def validate_cidr(s):
    """Validate a CIDR notation ip address.

    The string is considered a valid CIDR address if it consists of a valid
    IPv6 address in hextet format followed by a forward slash (/) and a bit
    mask length (0-128).

    >>> validate_cidr('::/128')
    True
    >>> validate_cidr('::/0')
    True
    >>> validate_cidr('fc00::/7')
    True
    >>> validate_cidr('::ffff:0:0/96')
    True
    >>> validate_cidr('::')
    False
    >>> validate_cidr('::/129')
    False
    >>> validate_cidr(None) #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: expected string or buffer

    :param s: String to validate as a CIDR notation ip address.
    :type s: str
    :returns: ``True`` if a valid CIDR address, ``False`` otherwise.
    :raises: TypeError
    """
    if _CIDR_RE.match(s):
        ip, mask = s.split('/')
        if validate_ip(ip):
            if int(mask) > 128:
                return False
        else:
            return False
        return True
    return False
#end validate_cidr

def cidr2block(cidr):
    """Convert a CIDR notation ip address into a tuple containing the network
    block start and end addresses.

    >>> cidr2block('2001:db8::/48')
    ('2001:db8::', '2001:db8:0:ffff:ffff:ffff:ffff:ffff')
    >>> cidr2block('::/0')
    ('::', 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff')

    :param cidr: CIDR notation ip address (eg. '127.0.0.1/8').
    :type cidr: str
    :returns: Tuple of block (start, end) or ``None`` if invalid.
    :raises: TypeError
    """
    if not validate_cidr(cidr):
        return None

    ip, prefix = cidr.split('/')
    prefix = int(prefix)
    ip = ip2long(ip)

    # keep left most prefix bits of ip
    shift = 128 - prefix
    block_start = ip >> shift << shift

    # expand right most 128 - prefix bits to 1
    mask = (1 << shift) - 1
    block_end = block_start | mask
    return (long2ip(block_start), long2ip(block_end))
#end cidr2block

# vim: set sw=4 ts=4 sts=4 et :
<commit_msg>Replace xrange() with range() for python 3.2 compat.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2008-2013, Bryan Davis
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#     - Redistributions of source code must retain the above copyright notice,
#     this list of conditions and the following disclaimer.
#     - Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

__all__ = (
    'cidr2block',
    'ip2long',
    'long2ip',
    'validate_cidr',
    'validate_ip',
    'MAX_IP',
    'MIN_IP',
)

import re
from . import ipv4

#: Regex for validating an IPv6 in hex notation
_HEX_RE = re.compile(r'^([0-9a-f]{0,4}:){2,7}[0-9a-f]{0,4}$')

#: Regex for validating an IPv6 in dotted-quad notation
_DOTTED_QUAD_RE = re.compile(r'^([0-9a-f]{0,4}:){2,6}(\d{1,3}\.){0,3}\d{1,3}$')

#: Regex for validating a CIDR network
_CIDR_RE = re.compile(r'^([0-9a-f]{0,4}:){2,7}[0-9a-f]{0,4}/\d{1,3}$')

#: Mamimum IPv6 integer
MAX_IP = 0xffffffffffffffffffffffffffffffff
#: Minimum IPv6 integer
MIN_IP = 0x0

def validate_ip(s):
    """Validate a hexidecimal IPv6 ip address.

    >>> validate_ip('::')
    True
    >>> validate_ip('::1')
    True
    >>> validate_ip('2001:db8:85a3::8a2e:370:7334')
    True
    >>> validate_ip('2001:db8:85a3:0:0:8a2e:370:7334')
    True
    >>> validate_ip('2001:0db8:85a3:0000:0000:8a2e:0370:7334')
    True
    >>> validate_ip('2001:db8::1:0:0:1')
    True
    >>> validate_ip('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff')
    True
    >>> validate_ip('::ffff:192.0.2.128')
    True
    >>> validate_ip('::ff::ff')
    False
    >>> validate_ip('::fffff')
    False
    >>> validate_ip('::ffff:192.0.2.300')
    False
    >>> validate_ip(None) #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: expected string or buffer

    :param s: String to validate as a hexidecimal IPv6 ip address.
    :type s: str
    :returns: ``True`` if a valid hexidecimal IPv6 ip address,
              ``False`` otherwise.
    :raises: TypeError
    """
    if _HEX_RE.match(s):
        return len(s.split('::')) <= 2
    if _DOTTED_QUAD_RE.match(s):
        halves = s.split('::')
        if len(halves) > 2:
            return False
        hextets = s.split(':')
        quads = hextets[-1].split('.')
        for q in quads:
            if int(q) > 255:
                return False
        return True
    return False
#end validate_ip

def ip2long(ip):
    """Convert a hexidecimal IPv6 address to a network byte order 128-bit
    integer.

    >>> ip2long('::') == 0
    True
    >>> ip2long('::1') == 1
    True
    >>> expect = 0x20010db885a3000000008a2e03707334
    >>> ip2long('2001:db8:85a3::8a2e:370:7334') == expect
    True
    >>> ip2long('2001:db8:85a3:0:0:8a2e:370:7334') == expect
    True
    >>> ip2long('2001:0db8:85a3:0000:0000:8a2e:0370:7334') == expect
    True
    >>> expect = 0x20010db8000000000001000000000001
    >>> ip2long('2001:db8::1:0:0:1') == expect
    True
    >>> expect = 281473902969472
    >>> ip2long('::ffff:192.0.2.128') == expect
    True
    >>> expect = 0xffffffffffffffffffffffffffffffff
    >>> ip2long('ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff') == expect
    True
    >>> ip2long('ff::ff::ff') == None
    True

    :param ip: Hexidecimal IPv6 address
    :type ip: str
    :returns: Network byte order 128-bit integer or ``None`` if ip is invalid.
    """
    if not validate_ip(ip):
        return None

    if '.' in ip:
        # convert IPv4 suffix to hex
        chunks = ip.split(':')
        v4_int = ipv4.ip2long(chunks.pop())
        if v4_int is None:
            return None
        chunks.append('%x' % ((v4_int >> 16) & 0xffff))
        chunks.append('%x' % (v4_int & 0xffff))
        ip = ':'.join(chunks)

    halves = ip.split('::')
    hextets = halves[0].split(':')
    if len(halves) == 2:
        h2 = halves[1].split(':')
        for z in range(8 - (len(hextets) + len(h2))):
            hextets.append('0')
        for h in h2:
            hextets.append(h)
    #end if

    lngip = 0
    for h in hextets:
        if '' == h:
            h = '0'
        lngip = (lngip << 16) | int(h, 16)
    return lngip
#end ip2long

def long2ip(l):
    """Convert a network byte order 128-bit integer to a canonical IPv6
    address.

    >>> long2ip(2130706433)
    '::7f00:1'
    >>> long2ip(42540766411282592856904266426630537217)
    '2001:db8::1:0:0:1'
    >>> long2ip(MIN_IP)
    '::'
    >>> long2ip(MAX_IP)
    'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff'
    >>> long2ip(None) #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: unsupported operand type(s) for >>: 'NoneType' and 'int'
    >>> long2ip(-1) #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: expected int between 0 and <really big int> inclusive
    >>> long2ip(MAX_IP + 1) #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: expected int between 0 and <really big int> inclusive

    :param l: Network byte order 128-bit integer.
    :type l: int
    :returns: Canonical IPv6 address (eg. '::1').
    :raises: TypeError
    """
    if MAX_IP < l or l < MIN_IP:
        raise TypeError(
            "expected int between %d and %d inclusive" % (MIN_IP, MAX_IP))

    # format as one big hex value
    hex_str = '%032x' % l
    # split into double octet chunks without padding zeros
    hextets = ['%x' % int(hex_str[x:x+4], 16) for x in range(0, 32, 4)]

    # find and remove left most longest run of zeros
    dc_start, dc_len = (-1, 0)
    run_start, run_len = (-1, 0)
    for idx, hextet in enumerate(hextets):
        if '0' == hextet:
            run_len += 1
            if -1 == run_start:
                run_start = idx
            if run_len > dc_len:
                dc_len, dc_start = (run_len, run_start)
        else:
            run_len, run_start = (0, -1)
    #end for
    if dc_len > 1:
        dc_end = dc_start + dc_len
        if dc_end == len(hextets):
            hextets += ['']
        hextets[dc_start:dc_end] = ['']
        if dc_start == 0:
            hextets = [''] + hextets
    #end if

    return ':'.join(hextets)
#end long2ip

def validate_cidr(s):
    """Validate a CIDR notation ip address.

    The string is considered a valid CIDR address if it consists of a valid
    IPv6 address in hextet format followed by a forward slash (/) and a bit
    mask length (0-128).

    >>> validate_cidr('::/128')
    True
    >>> validate_cidr('::/0')
    True
    >>> validate_cidr('fc00::/7')
    True
    >>> validate_cidr('::ffff:0:0/96')
    True
    >>> validate_cidr('::')
    False
    >>> validate_cidr('::/129')
    False
    >>> validate_cidr(None) #doctest: +IGNORE_EXCEPTION_DETAIL
    Traceback (most recent call last):
        ...
    TypeError: expected string or buffer

    :param s: String to validate as a CIDR notation ip address.
    :type s: str
    :returns: ``True`` if a valid CIDR address, ``False`` otherwise.
    :raises: TypeError
    """
    if _CIDR_RE.match(s):
        ip, mask = s.split('/')
        if validate_ip(ip):
            if int(mask) > 128:
                return False
        else:
            return False
        return True
    return False
#end validate_cidr

def cidr2block(cidr):
    """Convert a CIDR notation ip address into a tuple containing the network
    block start and end addresses.

    >>> cidr2block('2001:db8::/48')
    ('2001:db8::', '2001:db8:0:ffff:ffff:ffff:ffff:ffff')
    >>> cidr2block('::/0')
    ('::', 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff')

    :param cidr: CIDR notation ip address (eg. '127.0.0.1/8').
    :type cidr: str
    :returns: Tuple of block (start, end) or ``None`` if invalid.
    :raises: TypeError
    """
    if not validate_cidr(cidr):
        return None

    ip, prefix = cidr.split('/')
    prefix = int(prefix)
    ip = ip2long(ip)

    # keep left most prefix bits of ip
    shift = 128 - prefix
    block_start = ip >> shift << shift

    # expand right most 128 - prefix bits to 1
    mask = (1 << shift) - 1
    block_end = block_start | mask
    return (long2ip(block_start), long2ip(block_end))
#end cidr2block

# vim: set sw=4 ts=4 sts=4 et :
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
$Id$

Copyright (c) 2006-2012 sqlmap developers (http://www.sqlmap.org/)
See the file 'doc/COPYING' for copying permission
"""

import time

from lib.core.agent import agent
from lib.core.common import Backend
from lib.core.common import calculateDeltaSeconds
from lib.core.common import getCurrentThreadData
from lib.core.common import getUnicode
from lib.core.common import hashDBRetrieve
from lib.core.common import hashDBWrite
from lib.core.convert import base64unpickle
from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger
from lib.core.enums import DBMS
from lib.core.settings import SQL_STATEMENTS
from lib.core.settings import UNICODE_ENCODING
from lib.utils.timeout import timeout

def direct(query, content=True):
    select = True
    query = agent.payloadDirect(query)
    threadData = getCurrentThreadData()

    if Backend.isDbms(DBMS.ORACLE) and query.startswith("SELECT ") and " FROM " not in query:
        query = "%s FROM DUAL" % query

    for sqlTitle, sqlStatements in SQL_STATEMENTS.items():
        for sqlStatement in sqlStatements:
            if query.lower().startswith(sqlStatement) and sqlTitle != "SQL SELECT statement":
                select = False
                break

    if select and not query.upper().startswith("SELECT "):
        query = "SELECT " + query

    logger.log(9, query)

    output = hashDBRetrieve(query, True, True)

    start = time.time()
    if not select and "EXEC " not in query:
        _ = timeout(func=conf.dbmsConnector.execute, args=(query,), duration=conf.timeout, default=None)
    elif not (output and "sqlmapoutput" not in query and "sqlmapfile" not in query):
        output = timeout(func=conf.dbmsConnector.select, args=(query,), duration=conf.timeout, default=None)
        hashDBWrite(query, output, True)
    elif output:
        infoMsg = "resumed: %s..." % getUnicode(output, UNICODE_ENCODING)[:20]
        logger.info(infoMsg)
    threadData.lastQueryDuration = calculateDeltaSeconds(start)

    if not output:
        return output
    elif content:
        if output and isinstance(output, (list, tuple)):
            if len(output[0]) == 1:
                if len(output) > 1:
                    output = map(lambda _: _[0], output)
                else:
                    output = output[0][0]

        return getUnicode(output, noneToNull=True)
    else:
        for line in output:
            if line[0] in (1, -1):
                return True
            else:
                return False
<commit_msg>minor fix<commit_after>#!/usr/bin/env python

"""
$Id$

Copyright (c) 2006-2012 sqlmap developers (http://www.sqlmap.org/)
See the file 'doc/COPYING' for copying permission
"""

import time

from extra.safe2bin.safe2bin import safecharencode
from lib.core.agent import agent
from lib.core.common import Backend
from lib.core.common import calculateDeltaSeconds
from lib.core.common import getCurrentThreadData
from lib.core.common import getUnicode
from lib.core.common import hashDBRetrieve
from lib.core.common import hashDBWrite
from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger
from lib.core.enums import DBMS
from lib.core.settings import SQL_STATEMENTS
from lib.core.settings import UNICODE_ENCODING
from lib.utils.timeout import timeout

def direct(query, content=True):
    select = True
    query = agent.payloadDirect(query)
    threadData = getCurrentThreadData()

    if Backend.isDbms(DBMS.ORACLE) and query.startswith("SELECT ") and " FROM " not in query:
        query = "%s FROM DUAL" % query

    for sqlTitle, sqlStatements in SQL_STATEMENTS.items():
        for sqlStatement in sqlStatements:
            if query.lower().startswith(sqlStatement) and sqlTitle != "SQL SELECT statement":
                select = False
                break

    if select and not query.upper().startswith("SELECT "):
        query = "SELECT " + query

    logger.log(9, query)

    output = hashDBRetrieve(query, True, True)

    start = time.time()
    if not select and "EXEC " not in query:
        _ = timeout(func=conf.dbmsConnector.execute, args=(query,), duration=conf.timeout, default=None)
    elif not (output and "sqlmapoutput" not in query and "sqlmapfile" not in query):
        output = timeout(func=conf.dbmsConnector.select, args=(query,), duration=conf.timeout, default=None)
        hashDBWrite(query, output, True)
    elif output:
        infoMsg = "resumed: %s..." % getUnicode(output, UNICODE_ENCODING)[:20]
        logger.info(infoMsg)
    threadData.lastQueryDuration = calculateDeltaSeconds(start)

    if not output:
        return output
    elif content:
        if output and isinstance(output, (list, tuple)):
            if len(output[0]) == 1:
                if len(output) > 1:
                    output = map(lambda _: _[0], output)
                else:
                    output = output[0][0]

        retVal = getUnicode(output, noneToNull=True)
        return safecharencode(retVal) if kb.safeCharEncode else retVal
    else:
        for line in output:
            if line[0] in (1, -1):
                return True
            else:
                return False
<|endoftext|>


<commit_before>#
# Copyright (c) 2004 Specifix, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
"""
Implements changelog entries for repository commits.
"""

import os
import streams
import string
import tempfile

class AbstractChangeLog(streams.TupleStream):

    __slots__ = [ 'items' ]
    makeup = ( ("name",    streams.StringStream, "B"),
	       ("contact", streams.StringStream, "B"),
	       ("message", streams.StringStream, "!H") )

    def getName(self):
	return self.items[0].value()

    def setName(self, value):
	return self.items[0].set(value)

    def getContact(self):
	return self.items[1].value()

    def setContact(self, value):
	return self.items[1].set(value)

    def getMessage(self):
	return self.items[2].value()

    def setMessage(self, value):
	assert(value[-1] == '\n')
	return self.items[2].set(value)

    def freeze(self):
	if self.items[0].value() or self.items[1].value() or \
	   self.items[2].value():
	    return streams.TupleStream.freeze(self)
	else:
	    return ""

    def getMessageFromUser(self):
	editor = os.environ.get("EDITOR", "/bin/vi")
	(fd, name) = tempfile.mkstemp()
	msg = "\n-----\nEnter your change log message.\n"
	os.write(fd, msg)
	os.close(fd)

	os.system("%s %s" % (editor, name))

	newMsg = open(name).read()
	os.unlink(name)

	if newMsg == msg:
	    return False

	if newMsg[-len(msg):]:
	    newMsg = newMsg[:-len(msg)]

	newMsg = string.strip(newMsg)
	newMsg += '\n'
	self.setMessage(newMsg)
	return True

    def __eq__(self, other):
	if not isinstance(other, AbstractChangeLog):
	    return False

	return self.items == other.items

    def __init__(self, data = None):
	if data == "": data = None
	streams.TupleStream.__init__(self, data)

class ChangeLog(AbstractChangeLog):

    __slots__ = [ 'items' ]

    def __init__(self, name, contact, message):
	assert(not message or message[-1] == '\n')

	AbstractChangeLog.__init__(self)

	self.setName(name)
	self.setContact(contact)
	self.setMessage(message)

class ThawChangeLog(AbstractChangeLog):

    __slots__ = [ 'items' ]

    pass
<commit_msg>we ought to be able to instantiate changelog's with no message<commit_after>#
# Copyright (c) 2004 Specifix, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
"""
Implements changelog entries for repository commits.
"""

import os
import streams
import string
import tempfile

class AbstractChangeLog(streams.TupleStream):

    __slots__ = [ 'items' ]
    makeup = ( ("name",    streams.StringStream, "B"),
	       ("contact", streams.StringStream, "B"),
	       ("message", streams.StringStream, "!H") )

    def getName(self):
	return self.items[0].value()

    def setName(self, value):
	return self.items[0].set(value)

    def getContact(self):
	return self.items[1].value()

    def setContact(self, value):
	return self.items[1].set(value)

    def getMessage(self):
	return self.items[2].value()

    def setMessage(self, value):
	assert(not value or value[-1] == '\n')
	return self.items[2].set(value)

    def freeze(self):
	if self.items[0].value() or self.items[1].value() or \
	   self.items[2].value():
	    return streams.TupleStream.freeze(self)
	else:
	    return ""

    def getMessageFromUser(self):
	editor = os.environ.get("EDITOR", "/bin/vi")
	(fd, name) = tempfile.mkstemp()
	msg = "\n-----\nEnter your change log message.\n"
	os.write(fd, msg)
	os.close(fd)

	os.system("%s %s" % (editor, name))

	newMsg = open(name).read()
	os.unlink(name)

	if newMsg == msg:
	    return False

	if newMsg[-len(msg):]:
	    newMsg = newMsg[:-len(msg)]

	newMsg = string.strip(newMsg)
	newMsg += '\n'
	self.setMessage(newMsg)
	return True

    def __eq__(self, other):
	if not isinstance(other, AbstractChangeLog):
	    return False

	return self.items == other.items

    def __init__(self, data = None):
	if data == "": data = None
	streams.TupleStream.__init__(self, data)

class ChangeLog(AbstractChangeLog):

    __slots__ = [ 'items' ]

    def __init__(self, name, contact, message):
	assert(not message or message[-1] == '\n')

	AbstractChangeLog.__init__(self)

	self.setName(name)
	self.setContact(contact)
	self.setMessage(message)

class ThawChangeLog(AbstractChangeLog):

    __slots__ = [ 'items' ]

    pass
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

import os
import platform
import sys

path0 = os.path.dirname(os.path.abspath(__file__))
path1 = os.path.abspath(os.path.join(path0, os.pardir, 'util'))
sys.path.append(path1)
from subprocesses import envWithPath, captureStdout, isARMv7l, isMac, isWin, isMozBuild64, \
    normExpUserPath, shellify, vdump

if os.name == 'nt':
    COMPILE_NSPR_LIB = 'libnspr4.lib' if isMozBuild64 else 'nspr4.lib'
    COMPILE_PLDS_LIB = 'libplds4.lib' if isMozBuild64 else 'plds4.lib'
    COMPILE_PLC_LIB = 'libplc4.lib' if isMozBuild64 else 'plc4.lib'

    RUN_NSPR_LIB = 'libnspr4.dll' if isMozBuild64 else 'nspr4.dll'
    RUN_PLDS_LIB = 'libplds4.dll' if isMozBuild64 else 'plds4.dll'
    RUN_PLC_LIB = 'libplc4.dll' if isMozBuild64 else 'plc4.dll'
else:
    COMPILE_NSPR_LIB = 'libnspr4.a'
    COMPILE_PLDS_LIB = 'libplds4.a'
    COMPILE_PLC_LIB = 'libplc4.a'

    if platform.system() == 'Darwin':
        RUN_NSPR_LIB = 'libnspr4.dylib'
        RUN_PLDS_LIB = 'libplds4.dylib'
        RUN_PLC_LIB = 'libplc4.dylib'
    elif (platform.system() == 'Linux'):
        RUN_NSPR_LIB = 'libnspr4.so'
        RUN_PLDS_LIB = 'libplds4.so'
        RUN_PLC_LIB = 'libplc4.so'

ALL_COMPILE_LIBS = (COMPILE_NSPR_LIB, COMPILE_PLDS_LIB, COMPILE_PLC_LIB)
ALL_RUN_LIBS = (RUN_NSPR_LIB, RUN_PLDS_LIB, RUN_PLC_LIB)

def archOfBinary(binary):
    '''This function tests if a binary is 32-bit or 64-bit.'''
    unsplitFiletype = captureStdout(['file', binary])[0]
    filetype = unsplitFiletype.split(':', 1)[1]
    if isWin:
        assert 'PE executable for MS Windows (console)' in filetype
        return '32' if 'Intel 80386 32-bit' in filetype else '64'
    else:
        if 'universal binary' in filetype:
            raise Exception("I don't know how to deal with multiple-architecture binaries")
        if '32-bit' in filetype or 'i386' in filetype:
            assert '64-bit' not in filetype
            return '32'
        if '64-bit' in filetype:
            assert '32-bit' not in filetype
            return '64'

def constructVgCmdList(errorCode=77):
    '''Constructs default parameters needed to run valgrind with.'''
    vgCmdList = []
    vgCmdList.append('valgrind')
    if isMac:
        vgCmdList.append('--dsymutil=yes')
    vgCmdList.append('--error-exitcode=' + str(errorCode))
    vgCmdList.append('--smc-check=all-non-file')
    # See bug 913876 comment 18:
    vgCmdList.append('--vex-iropt-register-updates=allregs-at-mem-access')
    vgCmdList.append('--gen-suppressions=all')
    vgCmdList.append('--leak-check=full')
    vgCmdList.append('--show-possibly-lost=no')
    vgCmdList.append('--num-callers=50')
    return vgCmdList

def shellSupports(shellPath, args):
    '''
    This function returns True if the shell likes the args.
    You can support for a function, e.g. ['-e', 'foo()'], or a flag, e.g. ['-j', '-e', '42'].
    '''
    output, retCode = testBinary(shellPath, args, False, False)
    if retCode == 0:
        return True
    elif 1 <= retCode <= 3:
        # Exit codes 1 through 3 are all plausible "non-support":
        #   * "Usage error" is 1 in new js shell, 2 in old js shell, 2 in xpcshell.
        #   * "Script threw an error" is 3 in most shells, but 1 in some versions (see bug 751425).
        # Since we want autoBisect to support all shell versions, allow all these exit codes.
        return False
    else:
        raise Exception('Unexpected exit code in shellSupports ' + str(retCode))

def testBinary(shellPath, args, useValgrind, threadsafeShell):
    '''Tests the given shell with the given args.'''
    testCmd = (constructVgCmdList() if useValgrind else []) + [shellPath] + args
    vdump('The testing command is: ' + shellify(testCmd))

    newEnv = envWithPath(os.path.dirname(os.path.abspath(shellPath)))
    if threadsafeShell:
        # The NSPR libraries needed to run threadsafe js shell should have already been be copied to
        # the same destination as the shell.
        assert os.path.isfile(normExpUserPath(os.path.join(
            os.path.dirname(os.path.abspath(shellPath)), RUN_NSPR_LIB)))
        assert os.path.isfile(normExpUserPath(os.path.join(
            os.path.dirname(os.path.abspath(shellPath)), RUN_PLDS_LIB)))
        assert os.path.isfile(normExpUserPath(os.path.join(
            os.path.dirname(os.path.abspath(shellPath)), RUN_PLC_LIB)))
    out, rCode = captureStdout(testCmd, combineStderr=True, ignoreStderr=True, ignoreExitCode=True,
                               env=newEnv)
    vdump('The exit code is: ' + str(rCode))
    return out, rCode

def testDbgOrOpt(s):
    '''This function tests if a binary is a debug or optimized shell.'''
    # Do not use disassemble(), old shells prior to cc4fdccc1135 did not have disassemble(), and
    # it landed fairly recently on March 31, 2011. See bug 396512 comment 36.
    # The changeset's patch date is not reflective of its actual landing date.
    return 'dbg' if shellSupports(s, ['-e', 'dis()']) else 'opt'

def testGetBuildConfiguration(s):
    '''This function tests if a binary supports getBuildConfiguration().'''
    return shellSupports(s, ['-e', 'getBuildConfiguration()'])

def testGetBuildConfigurationWithThreadsafe(s):
    '''
    This function tests if a binary supports getBuildConfiguration() with threadsafe.
    See bug 791146 - getBuildConfiguration() returns the wrong value for gczeal and threadsafe
    '''
    ans = testBinary(s,
            ['-e', 'print(getBuildConfiguration().hasOwnProperty("threadsafe"))'], False, False)[0]
    return ans.find('true') != -1

def testJsShellOrXpcshell(s):
    '''This function tests if a binary is a js shell or xpcshell.'''
    return 'xpcshell' if shellSupports(s, ['-e', 'Components']) else 'jsShell'

def queryBuildConfiguration(s, parameter):
    '''Tests if a binary is compiled with specified parameters, in getBuildConfiguration().'''
    ans = testBinary(s, ['-e', 'print(getBuildConfiguration()["' + parameter + '"])'],
                     False, False)[0]
    return ans.find('true') != -1

def testIsHardFpShellARM(s):
    '''Tests if the ARM shell is compiled with hardfp support.'''
    readelfBin = '/usr/bin/readelf'
    if os.path.exists(readelfBin):
        newEnv = envWithPath(os.path.dirname(os.path.abspath(s)))
        readelfOutput = captureStdout([readelfBin, '-A', s], env=newEnv)[0]
        return ('Tag_ABI_VFP_args: VFP registers' in readelfOutput)
    else:
        raise Exception('readelf is not found.')

def verifyBinary(sh, options):
    '''Verifies that the binary is compiled as intended.'''
    assert archOfBinary(sh.getShellBaseTempDirWithName()) == sh.buildOptions.arch
    assert testDbgOrOpt(sh.getShellBaseTempDirWithName()) == sh.buildOptions.compileType
    if isARMv7l:
        assert testIsHardFpShellARM(sh.getShellBaseTempDirWithName()) == options.enableHardFp
    if testGetBuildConfiguration(sh.getShellBaseTempDirWithName()):
        if testGetBuildConfigurationWithThreadsafe(sh.getShellBaseTempDirWithName()):
            assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'threadsafe') == \
                options.isThreadsafe
        assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'more-deterministic') == \
            options.enableMoreDeterministic
        assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'asan') == \
            options.buildWithAsan
        assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'rooting-analysis') == \
            options.enableRootAnalysis
        assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'exact-rooting') == \
            options.enableExactRooting
        assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'generational-gc') == \
            options.enableGcGenerational
<commit_msg>Replace Valgrind flag --vex-iropt-register-updates=allregs-at-mem-access with --vex-iropt-register-updates=allregs-at-each-insn.<commit_after>#!/usr/bin/env python
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

import os
import platform
import sys

path0 = os.path.dirname(os.path.abspath(__file__))
path1 = os.path.abspath(os.path.join(path0, os.pardir, 'util'))
sys.path.append(path1)
from subprocesses import envWithPath, captureStdout, isARMv7l, isMac, isWin, isMozBuild64, \
    normExpUserPath, shellify, vdump

if os.name == 'nt':
    COMPILE_NSPR_LIB = 'libnspr4.lib' if isMozBuild64 else 'nspr4.lib'
    COMPILE_PLDS_LIB = 'libplds4.lib' if isMozBuild64 else 'plds4.lib'
    COMPILE_PLC_LIB = 'libplc4.lib' if isMozBuild64 else 'plc4.lib'

    RUN_NSPR_LIB = 'libnspr4.dll' if isMozBuild64 else 'nspr4.dll'
    RUN_PLDS_LIB = 'libplds4.dll' if isMozBuild64 else 'plds4.dll'
    RUN_PLC_LIB = 'libplc4.dll' if isMozBuild64 else 'plc4.dll'
else:
    COMPILE_NSPR_LIB = 'libnspr4.a'
    COMPILE_PLDS_LIB = 'libplds4.a'
    COMPILE_PLC_LIB = 'libplc4.a'

    if platform.system() == 'Darwin':
        RUN_NSPR_LIB = 'libnspr4.dylib'
        RUN_PLDS_LIB = 'libplds4.dylib'
        RUN_PLC_LIB = 'libplc4.dylib'
    elif (platform.system() == 'Linux'):
        RUN_NSPR_LIB = 'libnspr4.so'
        RUN_PLDS_LIB = 'libplds4.so'
        RUN_PLC_LIB = 'libplc4.so'

ALL_COMPILE_LIBS = (COMPILE_NSPR_LIB, COMPILE_PLDS_LIB, COMPILE_PLC_LIB)
ALL_RUN_LIBS = (RUN_NSPR_LIB, RUN_PLDS_LIB, RUN_PLC_LIB)

def archOfBinary(binary):
    '''This function tests if a binary is 32-bit or 64-bit.'''
    unsplitFiletype = captureStdout(['file', binary])[0]
    filetype = unsplitFiletype.split(':', 1)[1]
    if isWin:
        assert 'PE executable for MS Windows (console)' in filetype
        return '32' if 'Intel 80386 32-bit' in filetype else '64'
    else:
        if 'universal binary' in filetype:
            raise Exception("I don't know how to deal with multiple-architecture binaries")
        if '32-bit' in filetype or 'i386' in filetype:
            assert '64-bit' not in filetype
            return '32'
        if '64-bit' in filetype:
            assert '32-bit' not in filetype
            return '64'

def constructVgCmdList(errorCode=77):
    '''Constructs default parameters needed to run valgrind with.'''
    vgCmdList = []
    vgCmdList.append('valgrind')
    if isMac:
        vgCmdList.append('--dsymutil=yes')
    vgCmdList.append('--error-exitcode=' + str(errorCode))
    vgCmdList.append('--smc-check=all-non-file')
    # See bug 913876 comment 18:
    vgCmdList.append('--vex-iropt-register-updates=allregs-at-each-insn')
    vgCmdList.append('--gen-suppressions=all')
    vgCmdList.append('--leak-check=full')
    vgCmdList.append('--show-possibly-lost=no')
    vgCmdList.append('--num-callers=50')
    return vgCmdList

def shellSupports(shellPath, args):
    '''
    This function returns True if the shell likes the args.
    You can support for a function, e.g. ['-e', 'foo()'], or a flag, e.g. ['-j', '-e', '42'].
    '''
    output, retCode = testBinary(shellPath, args, False, False)
    if retCode == 0:
        return True
    elif 1 <= retCode <= 3:
        # Exit codes 1 through 3 are all plausible "non-support":
        #   * "Usage error" is 1 in new js shell, 2 in old js shell, 2 in xpcshell.
        #   * "Script threw an error" is 3 in most shells, but 1 in some versions (see bug 751425).
        # Since we want autoBisect to support all shell versions, allow all these exit codes.
        return False
    else:
        raise Exception('Unexpected exit code in shellSupports ' + str(retCode))

def testBinary(shellPath, args, useValgrind, threadsafeShell):
    '''Tests the given shell with the given args.'''
    testCmd = (constructVgCmdList() if useValgrind else []) + [shellPath] + args
    vdump('The testing command is: ' + shellify(testCmd))

    newEnv = envWithPath(os.path.dirname(os.path.abspath(shellPath)))
    if threadsafeShell:
        # The NSPR libraries needed to run threadsafe js shell should have already been be copied to
        # the same destination as the shell.
        assert os.path.isfile(normExpUserPath(os.path.join(
            os.path.dirname(os.path.abspath(shellPath)), RUN_NSPR_LIB)))
        assert os.path.isfile(normExpUserPath(os.path.join(
            os.path.dirname(os.path.abspath(shellPath)), RUN_PLDS_LIB)))
        assert os.path.isfile(normExpUserPath(os.path.join(
            os.path.dirname(os.path.abspath(shellPath)), RUN_PLC_LIB)))
    out, rCode = captureStdout(testCmd, combineStderr=True, ignoreStderr=True, ignoreExitCode=True,
                               env=newEnv)
    vdump('The exit code is: ' + str(rCode))
    return out, rCode

def testDbgOrOpt(s):
    '''This function tests if a binary is a debug or optimized shell.'''
    # Do not use disassemble(), old shells prior to cc4fdccc1135 did not have disassemble(), and
    # it landed fairly recently on March 31, 2011. See bug 396512 comment 36.
    # The changeset's patch date is not reflective of its actual landing date.
    return 'dbg' if shellSupports(s, ['-e', 'dis()']) else 'opt'

def testGetBuildConfiguration(s):
    '''This function tests if a binary supports getBuildConfiguration().'''
    return shellSupports(s, ['-e', 'getBuildConfiguration()'])

def testGetBuildConfigurationWithThreadsafe(s):
    '''
    This function tests if a binary supports getBuildConfiguration() with threadsafe.
    See bug 791146 - getBuildConfiguration() returns the wrong value for gczeal and threadsafe
    '''
    ans = testBinary(s,
            ['-e', 'print(getBuildConfiguration().hasOwnProperty("threadsafe"))'], False, False)[0]
    return ans.find('true') != -1

def testJsShellOrXpcshell(s):
    '''This function tests if a binary is a js shell or xpcshell.'''
    return 'xpcshell' if shellSupports(s, ['-e', 'Components']) else 'jsShell'

def queryBuildConfiguration(s, parameter):
    '''Tests if a binary is compiled with specified parameters, in getBuildConfiguration().'''
    ans = testBinary(s, ['-e', 'print(getBuildConfiguration()["' + parameter + '"])'],
                     False, False)[0]
    return ans.find('true') != -1

def testIsHardFpShellARM(s):
    '''Tests if the ARM shell is compiled with hardfp support.'''
    readelfBin = '/usr/bin/readelf'
    if os.path.exists(readelfBin):
        newEnv = envWithPath(os.path.dirname(os.path.abspath(s)))
        readelfOutput = captureStdout([readelfBin, '-A', s], env=newEnv)[0]
        return ('Tag_ABI_VFP_args: VFP registers' in readelfOutput)
    else:
        raise Exception('readelf is not found.')

def verifyBinary(sh, options):
    '''Verifies that the binary is compiled as intended.'''
    assert archOfBinary(sh.getShellBaseTempDirWithName()) == sh.buildOptions.arch
    assert testDbgOrOpt(sh.getShellBaseTempDirWithName()) == sh.buildOptions.compileType
    if isARMv7l:
        assert testIsHardFpShellARM(sh.getShellBaseTempDirWithName()) == options.enableHardFp
    if testGetBuildConfiguration(sh.getShellBaseTempDirWithName()):
        if testGetBuildConfigurationWithThreadsafe(sh.getShellBaseTempDirWithName()):
            assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'threadsafe') == \
                options.isThreadsafe
        assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'more-deterministic') == \
            options.enableMoreDeterministic
        assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'asan') == \
            options.buildWithAsan
        assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'rooting-analysis') == \
            options.enableRootAnalysis
        assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'exact-rooting') == \
            options.enableExactRooting
        assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'generational-gc') == \
            options.enableGcGenerational
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
charm-cli.py: Simple command line interface for CHarm.
"""

import argparse
import logging

try:
    import matplotlib

    matplotlib.use('Agg')
    matplotlib.rc('font', **{'sans-serif': 'DejaVu Sans',
                             'serif': 'DejaVu Serif',
                             'family': 'sans-serif'})
    import matplotlib.pyplot
except ImportError as e:
    print('ERROR: {}'.format(e.msg))
    exit(1)
try:
    import numpy
except ImportError as e:
    print('ERROR: {}'.format(e.msg))
    exit(1)

try:
    from LibCharm.Sequence import Sequence
    from LibCharm import IO
except ImportError as e:
    print('ERROR: {}'.format(e.msg))
    exit(1)

def autolabel(rects, ax, labels, vertical=True):
    """
    Automatically adds labels above a bar in a bar graph.

    :param rects:    list of bars to be labelled (e.g. generated by ax.bar())
    :param ax:       axis (axis object from matplotlib)
    :param labels:   list of labels
    :param vertical: rotate the labels by 90° if true

    """
    if vertical:
        rotation = 'vertical'
    else:
        rotation = 'horizontal'

    if len(labels) == len(rects):
        heights = []
        for rect in rects:
            height = rect.get_height()
            heights.append(height)

        max_height = max(heights)

        for rect in rects:
            i = rects.index(rect)
            label = labels[i]
            height = rect.get_height()
            if height > 0:
                y = 1.05 * height
            else:
                y = 0.02 * max_height
            ax.text(rect.get_x() + rect.get_width() / 2., y, str(label),
                    ha='center', va='bottom', rotation=rotation, size='x-small')

def plot_codon_usage(sequence, ax):
    """
    Plot the codon usage for origin and target host as bar graph

    :param sequence: LibCharm.Sequence object
    :param ax      : matplotlib axis object
    """

    x1 = x2 = numpy.arange(len(sequence.codons))
    bar_width = 0.5
    xlabels = []

    origin_f = []
    target_f = []

    # extract data to plot from sequence object
    for c in sequence.codons:
        origin_f.append(c['origin_f'])
        target_f.append(c['target_f'])
        xlabels.append(c['aa'])

    # convert lists to numpy arrays
    origin_f = numpy.array(origin_f)
    target_f = numpy.array(target_f)

    # plot data
    p1 = ax.bar(x1, origin_f, color='b', width=bar_width)
    p2 = ax.bar(x2 + (0.5 * bar_width), target_f, color='r', width=bar_width)

    # hide top and right axes
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    # set tick parameters
    ax.tick_params(axis='both', which='both', direction='out')
    ax.get_xaxis().tick_bottom()
    ax.get_yaxis().tick_left()
    # position xticks and labels on x axis to be centered for both bars
    ax.set_xticks(x1 + bar_width / 2)
    ax.set_xticklabels(xlabels, **{'family': 'monospace'})
    ax.set_xlabel('amino acid')
    # add a legend to the plot
    ax.legend((p1, p2), ('Origin organism', 'Host organism'), loc=2, bbox_to_anchor=(1, 1))
    ax.hlines(sequence.lower_threshold, 0, len(x1), colors='k', linestyles='solid', **{'linewidth': 1})

    if not sequence.use_frequency:
        # set the y axis label
        ax.set_ylabel('codon usage [fraction]')
        # specify the distance between the ticks on the y axis
        major_locator = matplotlib.ticker.MultipleLocator(0.1)
        minor_locator = matplotlib.ticker.MultipleLocator(0.01)
    else:
        # set the y axis label if frequency is used instead of fractions
        ax.set_ylabel('codon usage [frequency/1000]')
        # specify the distance between the ticks on the y axis
        major_locator = matplotlib.ticker.MultipleLocator(10)
        minor_locator = matplotlib.ticker.MultipleLocator(1)

    # set the distance between the ticks on the y axis
    ax.yaxis.set_major_locator(major_locator)
    ax.yaxis.set_minor_locator(minor_locator)

def plot_codon_usage_differences(sequence, ax):
    """
    Plot the difference in codon usage for origin and target host as bar graph

    :param sequence: LibCharm.Sequence object
    :param ax:       matplotlib axis object
    """

    # Generate a range of residues out of the length of the sequence array
    x1 = numpy.arange(len(sequence.codons))

    # Set the threshold according to use_frequency
    if sequence.use_frequency:
        threshold = 5
    else:
        threshold = 0.2

    # Set width of bars
    bar_width = 0.8
    # Initialize array of labels for the x axis
    xlabels = []

    # Initialize arrays of data and labels for the bars
    df = []
    bar_labels = []

    # walk over the codons in sequence
    for c in sequence.codons:
        # add final_df to data array
        df.append(c['final_df'])
        # add residue to xlabels
        xlabels.append(c['aa'])
        # generate bar label and add to list
        label = u'{} → {}'.format(c['original'], c['new'])
        bar_labels.append(label)

    # convert lists to numpy arrays
    bar_labels = numpy.array(bar_labels)
    df = numpy.array(df)
    # find bars that exceed the threshold
    mask1 = numpy.ma.where(df > threshold)
    mask2 = numpy.ma.where(df <= threshold)

    # plot and color bars accordingly
    p1 = ax.bar(x1[mask1], df[mask1], color='r', width=bar_width)
    autolabel(p1, ax, bar_labels[mask1], vertical=True)

    p2 = ax.bar(x1[mask2], df[mask2], color='b', width=bar_width)
    autolabel(p2, ax, bar_labels[mask2], vertical=True)

    # hide top and right axis
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    ax.tick_params(axis='both', which='both', direction='out')
    ax.get_xaxis().tick_bottom()
    ax.get_yaxis().tick_left()

    # set x axis labels to be centered and to use a monospaced font
    ax.set_xticks(x1 + bar_width / 2)
    ax.set_xticklabels(xlabels, **{'family': 'monospace'})
    ax.set_xlabel('amino acid')

    ax.set_ylabel(r'Differential codon usage $f_{origin} - f_{host}$')

    if not sequence.use_frequency:
        major_locator = matplotlib.ticker.MultipleLocator(0.05)
        minor_locator = matplotlib.ticker.MultipleLocator(0.01)
    else:
        major_locator = matplotlib.ticker.MultipleLocator(10)
        minor_locator = matplotlib.ticker.MultipleLocator(1)

    ax.legend((p1, p2), (u'df > {}'.format(threshold), u'df ≤ {}'.format(threshold)), loc=2, bbox_to_anchor=(1, 1))

    ax.yaxis.set_major_locator(major_locator)
    ax.yaxis.set_minor_locator(minor_locator)

    ax.hlines(threshold, 0, len(x1), colors='k', linestyles='dotted', **{'linewidth': 1})

def plot(sequence, prefix=None):
    """
    Wrapper for plot_codon_usage_differences and plot_codon_usage

    :param sequence:  LibCharm.Sequence object
    :param prefix:    Resulting plot files will be prefixed with 'prefix'
    """

    if prefix:
        filename = '{}_charm_results.svg'.format(prefix)
    else:
        filename = 'charm_results.svg'

    # Create a plot with two subplots
    fig, axarr = matplotlib.pyplot.subplots(2, figsize=(50, 20), dpi=300)

    # Actually plot data
    plot_codon_usage(sequence, axarr[0])
    plot_codon_usage_differences(sequence, axarr[1])

    # Save plot as svg
    matplotlib.pyplot.savefig(filename, format='svg', orientation='landscape', papertype='a4')

def parse_arguments():
    """
    Parse command line arguments and return list of arguments
    """
    parser = argparse.ArgumentParser()
    parser.add_argument('-v', '--verbose', action='store_true', help='increase output verbosity')
    parser.add_argument('-p', '--prefix', type=str, help='prefix for output files')
    parser.add_argument('-f', '--frequency', action='store_true', help='use frequency/1000 instead of fraction')
    parser.add_argument('-l', '--lower_frequency_alternative', action='store_true',
                        help='if two codons result in the same difference in codon usage '
                             'between origin and target host, use the lower frequency alternative')
    parser.add_argument('-t', '--threshold', type=float,
                        help='Lower threshold of codon usage. Defaults to 0.1 and 5 for fraction and '
                             'frequency respectively')
    parser.add_argument('-to', '--translation_table_origin', type=int,
                        help='id of translation table; Default is: standard genetic code = 1; '
                             'id corresponds to \'trans_table\' '
                             'on http://www.ncbi.nlm.nih.gov/Taxonomy/Utils/wprintgc.cgi')
    parser.add_argument('-th', '--translation_table_host', type=int,
                        help='id of translation table; Default is: standard genetic code = 1; '
                             'id corresponds to \'trans_table\' '
                             'on http://www.ncbi.nlm.nih.gov/Taxonomy/Utils/wprintgc.cgi')
    parser.add_argument('origin', type=int, help='species id of origin organism taken from '
                                                 '\'http://www.kazusa.or.jp/codon\' (e.g. \'83333\' for E. coli K12)')
    parser.add_argument('host', type=int, help='species id of host organism taken from '
                                               '\'http://www.kazusa.or.jp/codon\' (e.g. \'83333\' for E. coli K12)')
    parser.add_argument('input', type=str, help='input file in FASTA format')
    args = parser.parse_args()

    return args

def initialize_logger(prefix):
    """
    Initialization of logging subsystem. Two logging handlers are brought up:
    'fh' which logs to a log file and 'ch' which logs to standard output.

    :param prefix:  prefix that is added to the filename
    :return logger: return a logger instance
    """
    logger = logging.getLogger('charm-cli')
    logger.setLevel(logging.INFO)

    ch = logging.StreamHandler()
    ch.setLevel(logging.INFO)
    logger.addHandler(ch)

    try:
        if prefix:
            log_filename = '{}_charm-cli.log'.format(prefix)
        else:
            log_filename = 'charm-cli.log'
        fh = logging.FileHandler(log_filename, 'w')
        fh.setLevel(logging.INFO)
        logger.addHandler(fh)
    except IOError as error:
        logger.warning('WARNING: Cannot create log file! Run charm-cli from a directory to '
                       'which you have write access.')
        logger.warning(error.msg)
        pass

    return logger

def main():
    """
    Main function of charm-cli.py.
    """

    # Parse command line arguments
    args = parse_arguments()
    # Initialize logging
    logger = initialize_logger(args.prefix)

    # Set translation tables according to user input. Defaults to standard genetic code (table 1)
    if args.translation_table_origin:
        translation_table_origin = args.translation_table_origin
    else:
        translation_table_origin = 1

    if args.translation_table_host:
        translation_table_host = args.translation_table_host
    else:
        translation_table_host = 1

    # set threshold if provided by the user and otherwise fall back to defaults
    if args.threshold:
        lower_threshold = args.threshold
    elif args.frequency:
        lower_threshold = 5
    else:
        lower_threshold = 0.1

    # initialize Sequence object with user provided input
    sequence = Sequence(IO.load_file(args.input), args.origin, args.host,
                        translation_table_origin=translation_table_origin,
                        translation_table_host=translation_table_host,
                        use_frequency=args.frequency,
                        lower_threshold=lower_threshold,
                        lower_alternative=args.lower_frequency_alternative)

    # harmonize the provided sequence
    harmonized_codons = sequence.get_harmonized_codons()
    # check if input and output sequence are identical
    verify_sequence = sequence.verify_harmonized_sequence()

    # log summary to standard output and log file
    logger.info('SUMMARY:\n')
    if verify_sequence:
        text = 'Success! Translation of harmonized and original sequence match:\n\n' \
               '{}\n'.format(sequence.harmonized_translated_sequence)
        logger.info(text)
    else:
        logger.error('ERROR: Translations of harmonized and original sequence DO NOT match!')
    logger.info('Harmonized codons: {}\n'.format(len(harmonized_codons)))

    table_header = '{:<10} {:^3} {:^4}    {:^4} {:^7} {:>6} {:<7} {:>6}'.format('position', 'aa', 'orig', 'new',
                                                                                'initial', 'final', 'origin', 'target')
    logger.info(table_header)

    for c in sequence.codons:
        if c['ambiguous']:
            if str(c['original']) != str(c['new']):
                line = '{:<10} {:^3} {:<4} -> {:<4} {:<5.2f} -> {:<3.2f}  {:<5.2f} -> {:<3.2f} WARNING: Original ' \
                       'codon was ambiguous!'.format(c['position'], c['aa'], c['original'], c['new'],
                                                     c['initial_df'], c['final_df'], c['origin_f'], c['target_f'])
            else:
                line = '{:<10} {:^3} {:<12} {:<5.2f}          {:<5.2f} -> {:<3.2f} WARNING: Original ' \
                       'codon was ambiguous!'.format(c['position'], c['aa'], c['original'], c['initial_df'],
                                                     c['origin_f'], c['target_f'])
        else:
            if str(c['original']) != str(c['new']):
                line = '{:<10} {:^3} {:<4} -> {:<4} {:<5.2f} -> {:<3.2f}  {:<5.2f} -> {:<3.2f}'.format(c['position'],
                                                                                c['aa'],
                                                                                c['original'],
                                                                                c['new'],
                                                                                c['initial_df'],
                                                                                c['final_df'],
                                                                                c['origin_f'],
                                                                                c['target_f'])
            else:
                line = '{:<10} {:^3} {:<12} {:<5.2f}          {:<5.2f} -> {:<3.2f}'.format(c['position'],
                                                                                c['aa'],
                                                                                c['original'],
                                                                                c['initial_df'],
                                                                                c['origin_f'],
                                                                                c['target_f'])

        logger.info(line)

    logger.info('\nCodon-harmonized sequence:\n\n{}'.format(sequence.harmonized_sequence))

    plot(sequence, args.prefix)

    # Exit gracefully
    exit(0)

if __name__ == "__main__":
    main()

<commit_msg>Prepare for tagging release 0.1: Added comments to charm-cli.py and refactored code<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
charm-cli.py: Simple command line interface for CHarm.
"""

import argparse
import logging

try:
    import matplotlib

    matplotlib.use('Agg')
    matplotlib.rc('font', **{'sans-serif': 'DejaVu Sans',
                             'serif': 'DejaVu Serif',
                             'family': 'sans-serif'})
    import matplotlib.pyplot
except ImportError as e:
    print('ERROR: {}'.format(e.msg))
    exit(1)
try:
    import numpy
except ImportError as e:
    print('ERROR: {}'.format(e.msg))
    exit(1)

try:
    from LibCharm.Sequence import Sequence
    from LibCharm import IO
except ImportError as e:
    print('ERROR: {}'.format(e.msg))
    exit(1)

def autolabel(rects, ax, labels, vertical=True):
    """
    Automatically adds labels above a bar in a bar graph.

    :param rects:    list of bars to be labelled (e.g. generated by ax.bar())
    :param ax:       axis (axis object from matplotlib)
    :param labels:   list of labels
    :param vertical: rotate the labels by 90° if true

    """
    if vertical:
        rotation = 'vertical'
    else:
        rotation = 'horizontal'

    if len(labels) == len(rects):
        heights = []
        for rect in rects:
            height = rect.get_height()
            heights.append(height)

        max_height = max(heights)

        for rect in rects:
            i = rects.index(rect)
            label = labels[i]
            height = rect.get_height()
            if height > 0:
                y = 1.05 * height
            else:
                y = 0.02 * max_height
            ax.text(rect.get_x() + rect.get_width() / 2., y, str(label),
                    ha='center', va='bottom', rotation=rotation, size='x-small')

def plot_codon_usage(sequence, ax):
    """
    Plot the codon usage for origin and target host as bar graph

    :param sequence: LibCharm.Sequence object
    :param ax      : matplotlib axis object
    """

    x1 = x2 = numpy.arange(len(sequence.codons))
    bar_width = 0.5
    xlabels = []

    origin_f = []
    target_f = []

    # extract data to plot from sequence object
    for c in sequence.codons:
        origin_f.append(c['origin_f'])
        target_f.append(c['target_f'])
        xlabels.append(c['aa'])

    # convert lists to numpy arrays
    origin_f = numpy.array(origin_f)
    target_f = numpy.array(target_f)

    # plot data
    p1 = ax.bar(x1, origin_f, color='b', width=bar_width)
    p2 = ax.bar(x2 + (0.5 * bar_width), target_f, color='r', width=bar_width)

    # hide top and right axes
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    # set tick parameters
    ax.tick_params(axis='both', which='both', direction='out')
    ax.get_xaxis().tick_bottom()
    ax.get_yaxis().tick_left()
    # position xticks and labels on x axis to be centered for both bars
    ax.set_xticks(x1 + bar_width / 2)
    ax.set_xticklabels(xlabels, **{'family': 'monospace'})
    ax.set_xlabel('amino acid')
    # add a legend to the plot
    ax.legend((p1, p2), ('Origin organism', 'Host organism'), loc=2, bbox_to_anchor=(1, 1))
    ax.hlines(sequence.lower_threshold, 0, len(x1), colors='k', linestyles='solid', **{'linewidth': 1})

    if not sequence.use_frequency:
        # set the y axis label
        ax.set_ylabel('codon usage [fraction]')
        # specify the distance between the ticks on the y axis
        major_locator = matplotlib.ticker.MultipleLocator(0.1)
        minor_locator = matplotlib.ticker.MultipleLocator(0.01)
    else:
        # set the y axis label if frequency is used instead of fractions
        ax.set_ylabel('codon usage [frequency/1000]')
        # specify the distance between the ticks on the y axis
        major_locator = matplotlib.ticker.MultipleLocator(10)
        minor_locator = matplotlib.ticker.MultipleLocator(1)

    # set the distance between the ticks on the y axis
    ax.yaxis.set_major_locator(major_locator)
    ax.yaxis.set_minor_locator(minor_locator)

def plot_codon_usage_differences(sequence, ax):
    """
    Plot the difference in codon usage for origin and target host as bar graph

    :param sequence: LibCharm.Sequence object
    :param ax:       matplotlib axis object
    """

    # Generate a range of residues out of the length of the sequence array
    x1 = numpy.arange(len(sequence.codons))

    # Set the threshold according to use_frequency
    if sequence.use_frequency:
        threshold = 5
    else:
        threshold = 0.2

    # Set width of bars
    bar_width = 0.8
    # Initialize array of labels for the x axis
    xlabels = []

    # Initialize arrays of data and labels for the bars
    df = []
    bar_labels = []

    # walk over the codons in sequence
    for c in sequence.codons:
        # add final_df to data array
        df.append(c['final_df'])
        # add residue to xlabels
        xlabels.append(c['aa'])
        # generate bar label and add to list
        label = u'{} → {}'.format(c['original'], c['new'])
        bar_labels.append(label)

    # convert lists to numpy arrays
    bar_labels = numpy.array(bar_labels)
    df = numpy.array(df)
    # find bars that exceed the threshold
    mask1 = numpy.ma.where(df > threshold)
    mask2 = numpy.ma.where(df <= threshold)

    # plot and color bars accordingly
    p1 = ax.bar(x1[mask1], df[mask1], color='r', width=bar_width)
    autolabel(p1, ax, bar_labels[mask1], vertical=True)

    p2 = ax.bar(x1[mask2], df[mask2], color='b', width=bar_width)
    autolabel(p2, ax, bar_labels[mask2], vertical=True)

    # hide top and right axis
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    ax.tick_params(axis='both', which='both', direction='out')
    ax.get_xaxis().tick_bottom()
    ax.get_yaxis().tick_left()

    # set x axis labels to be centered and to use a monospaced font
    ax.set_xticks(x1 + bar_width / 2)
    ax.set_xticklabels(xlabels, **{'family': 'monospace'})
    ax.set_xlabel('amino acid')

    ax.set_ylabel(r'Differential codon usage $f_{origin} - f_{host}$')

    if not sequence.use_frequency:
        major_locator = matplotlib.ticker.MultipleLocator(0.05)
        minor_locator = matplotlib.ticker.MultipleLocator(0.01)
    else:
        major_locator = matplotlib.ticker.MultipleLocator(10)
        minor_locator = matplotlib.ticker.MultipleLocator(1)

    ax.legend((p1, p2), (u'df > {}'.format(threshold), u'df ≤ {}'.format(threshold)), loc=2, bbox_to_anchor=(1, 1))

    ax.yaxis.set_major_locator(major_locator)
    ax.yaxis.set_minor_locator(minor_locator)

    ax.hlines(threshold, 0, len(x1), colors='k', linestyles='dotted', **{'linewidth': 1})

def plot(sequence, prefix=None):
    """
    Wrapper for plot_codon_usage_differences and plot_codon_usage

    :param sequence:  LibCharm.Sequence object
    :param prefix:    Resulting plot files will be prefixed with 'prefix'
    """

    if prefix:
        filename = '{}_charm_results.svg'.format(prefix)
    else:
        filename = 'charm_results.svg'

    # Create a plot with two subplots
    fig, axarr = matplotlib.pyplot.subplots(2, figsize=(50, 20), dpi=300)

    # Actually plot data
    plot_codon_usage(sequence, axarr[0])
    plot_codon_usage_differences(sequence, axarr[1])

    # Save plot as svg
    matplotlib.pyplot.savefig(filename, format='svg', orientation='landscape', papertype='a4')

def parse_arguments():
    """
    Parse command line arguments and return list of arguments
    """
    parser = argparse.ArgumentParser()
    parser.add_argument('-v', '--verbose', action='store_true', help='increase output verbosity')
    parser.add_argument('-p', '--prefix', type=str, help='prefix for output files')
    parser.add_argument('-f', '--frequency', action='store_true', help='use frequency/1000 instead of fraction')
    parser.add_argument('-l', '--lower_frequency_alternative', action='store_true',
                        help='if two codons result in the same difference in codon usage '
                             'between origin and target host, use the lower frequency alternative')
    parser.add_argument('-t', '--threshold', type=float,
                        help='Lower threshold of codon usage. Defaults to 0.1 and 5 for fraction and '
                             'frequency respectively')
    parser.add_argument('-to', '--translation_table_origin', type=int,
                        help='id of translation table; Default is: standard genetic code = 1; '
                             'id corresponds to \'trans_table\' '
                             'on http://www.ncbi.nlm.nih.gov/Taxonomy/Utils/wprintgc.cgi')
    parser.add_argument('-th', '--translation_table_host', type=int,
                        help='id of translation table; Default is: standard genetic code = 1; '
                             'id corresponds to \'trans_table\' '
                             'on http://www.ncbi.nlm.nih.gov/Taxonomy/Utils/wprintgc.cgi')
    parser.add_argument('origin', type=int, help='species id of origin organism taken from '
                                                 '\'http://www.kazusa.or.jp/codon\' (e.g. \'83333\' for E. coli K12)')
    parser.add_argument('host', type=int, help='species id of host organism taken from '
                                               '\'http://www.kazusa.or.jp/codon\' (e.g. \'83333\' for E. coli K12)')
    parser.add_argument('input', type=str, help='input file in FASTA format')
    args = parser.parse_args()

    return args

def initialize_logger(prefix):
    """
    Initialization of logging subsystem. Two logging handlers are brought up:
    'fh' which logs to a log file and 'ch' which logs to standard output.

    :param prefix:  prefix that is added to the filename
    :return logger: return a logger instance
    """
    logger = logging.getLogger('charm-cli')
    logger.setLevel(logging.INFO)

    ch = logging.StreamHandler()
    ch.setLevel(logging.INFO)
    logger.addHandler(ch)

    try:
        if prefix:
            log_filename = '{}_charm-cli.log'.format(prefix)
        else:
            log_filename = 'charm-cli.log'
        fh = logging.FileHandler(log_filename, 'w')
        fh.setLevel(logging.INFO)
        logger.addHandler(fh)
    except IOError as error:
        logger.warning('WARNING: Cannot create log file! Run charm-cli from a directory to '
                       'which you have write access.')
        logger.warning(error.msg)
        pass

    return logger

def main():
    """
    Main function of charm-cli.py.
    """

    # Parse command line arguments
    args = parse_arguments()
    # Initialize logging
    logger = initialize_logger(args.prefix)

    # Set translation tables according to user input. Defaults to standard genetic code (table 1)
    if args.translation_table_origin:
        translation_table_origin = args.translation_table_origin
    else:
        translation_table_origin = 1

    if args.translation_table_host:
        translation_table_host = args.translation_table_host
    else:
        translation_table_host = 1

    # set threshold if provided by the user and otherwise fall back to defaults
    if args.threshold:
        lower_threshold = args.threshold
    elif args.frequency:
        lower_threshold = 5
    else:
        lower_threshold = 0.1

    # initialize Sequence object with user provided input
    sequence = Sequence(IO.load_file(args.input), args.origin, args.host,
                        translation_table_origin=translation_table_origin,
                        translation_table_host=translation_table_host,
                        use_frequency=args.frequency,
                        lower_threshold=lower_threshold,
                        lower_alternative=args.lower_frequency_alternative)

    # harmonize the provided sequence
    harmonized_codons = sequence.get_harmonized_codons()
    # check if input and output sequence are identical
    verify_sequence = sequence.verify_harmonized_sequence()

    # log summary to standard output and log file
    logger.info('SUMMARY:\n')
    if verify_sequence:
        text = 'Success! Translation of harmonized and original sequence match:\n\n' \
               '{}\n'.format(sequence.harmonized_translated_sequence)
        logger.info(text)
    else:
        logger.error('ERROR: Translations of harmonized and original sequence DO NOT match!')
    logger.info('Harmonized codons: {}\n'.format(len(harmonized_codons)))

    table_header = '{:<10} {:^3} {:^4}    {:^4} {:^7} {:>6} {:<7} {:>6}'.format('position', 'aa', 'orig', 'new',
                                                                                'initial', 'final', 'origin', 'target')
    logger.info(table_header)

    warnings = []

    # Iterate over all codons in the sequence and print some statistics and information
    for c in sequence.codons:
        if str(c['original']) != str(c['new']):
            line = '{:<10} {:^3} {:<4} -> {:<4} {:<5.2f} -> {:<3.2f}  {:<5.2f} -> {:<3.2f}'.format(c['position'],
                                                                                c['aa'],
                                                                                c['original'],
                                                                                c['new'],
                                                                                c['initial_df'],
                                                                                c['final_df'],
                                                                                c['origin_f'],
                                                                                c['target_f'])
        else:
            line = '{:<10} {:^3} {:<12} {:<5.2f}          {:<5.2f} -> {:<3.2f}'.format(c['position'],
                                                                                c['aa'],
                                                                                c['original'],
                                                                                c['initial_df'],
                                                                                c['origin_f'],
                                                                                c['target_f'])
        if c['ambiguous']:
            line += ' WARNING: Original codon is ambiguous!'
            warnings.append('Codon {} ({}) coding for {} is ambiguous! {} was chosen for the '
                            'harmonized sequence!'.format(c['position'],
                                                          c['original'],
                                                          c['aa'],
                                                          c['new']))

        logger.info(line)

    logger.info('\nCodon-harmonized sequence:\n\n{}'.format(sequence.harmonized_sequence))
    if warnings:
        logger.warn('\nWARNINGS OCCURRED DURING HARMONIZATION:\n')
        for warning in warnings:
            logger.warn(warning)

    plot(sequence, args.prefix)

    # Exit gracefully
    exit(0)

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before># Chat Aliasing plugin by Curs3d #
##################################
# Allows users to alias words,
# so that when they send a
# message in chat, it gets
# replaced by their specified
# word. The JSON file for this
# plugin is generated if not
# present. Set values to -1
# for "unlimited" setting.

from helpers import *
import re
from traceback import format_exc as trace

data = None

max_entries = 10
max_alias_length = 35
# Minecraft message limit is 100 so I decided to give a little tolerance (and I added a bit more)
max_overall_length = 100 + max_alias_length

alias_perm = "utils.alias.allowed"
exceed_length = "utils.alias.exceedlimit"
exceed_entries = "utils.alias.exceedlimit"
exceed_overall_length = "utils.alias.exceedlimit"

def safe_open_json():
    global data
    if data is not None:
        return data
    data = open_json_file("aliases")
    if data is None:
        data = {}
    save_json_file("aliases", data)
    return data

def multiple_replace(aliases, text):
    regex = re.compile("|".join(map(re.escape, aliases.keys())))
    return regex.sub(lambda mo: aliases[mo.group(0)], text)

@hook.command("alias",
              usage="/<command> [to_alias] [alias...]",
              desc="Aliases words in chat")
def on_alias_command(sender, cmd, label, args):

    if not is_player(sender):
        msg(sender, "Sorry, non-players cannot alias words")
        return True
    if not sender.hasPermission(alias_perm):
        plugin_header(recipient=sender, name="Chat Alias")
        noperm(sender)
        return True
    if len(args) == 0:
        plugin_header(recipient=sender, name="Chat Alias")
        msg(sender, "&7This is a plugin that allows you to get words" +
            "replaced by other ones automatically!")
        msg(sender, "&7\nCommands:")
        msg(sender, "&e/alias <word> &7removes <word> from your aliases. " +
            "Use &e/alias * &7to remove all aliases.")
        msg(sender, "&e/alias <word> <replacement> &7will change &e<word> " +
            "&7to &e<replacement> &7in chat")
        msg(sender, "&7\nYour Aliases:")
        data = safe_open_json()
        try:
            for alias, value in data[str(sender.getUniqueId())].items():
                msg(sender, "&7%s &7==> %s" % (alias, value))
        except KeyError:
            pass
        return True
    elif len(args) == 1:
        data = safe_open_json()
        if args[0] == "*":
            try:
                del data[str(sender.getUniqueId())]
            except KeyError:
                plugin_header(recipient=sender, name="Chat Alias")
                msg(sender, "&7No alias data to remove!")
                return True
            save_json_file("aliases", data)
            plugin_header(recipient=sender, name="Chat Alias")
            msg(sender, "&cALL &7alias data successfuly removed!")
            return True
        try:
            if data[str(sender.getUniqueId())].pop(args[0], None) is None:
                plugin_header(recipient=sender, name="Chat Alias")
                msg(sender, "&7Could not remove: alias not present!")
                return True
        except KeyError:
            plugin_header(recipient=sender, name="Chat Alias")
            msg(sender, "&7Could not remove: you do not have any aliases!")
            return True
        save_json_file("aliases", data)
        plugin_header(recipient=sender, name="Chat Alias")
        msg(sender, "&7Alias for %s &7successfuly removed" % args[0])
        return True
    elif len(args) >= 2:
        data = safe_open_json()
        alias = " ".join(args[1:])
        try:
            if (len(alias) > max_alias_length) and (max_alias_length >= 0) and (not sender.hasPermission(exceed_length)):
                plugin_header(recipient=sender, name="Chat Alias")
                msg(sender, "&7Please do not alias long words/sentences.")
                return True
            if (len(data[str(sender.getUniqueId())]) >= max_entries) and (max_entries >= 0) and (not sender.hasPermission(exceed_entries)):
                plugin_header(recipient=sender, name="Chat Alias")
                msg(sender, "&7You have reached your alias limit!")
                return True
        except KeyError:
            data[str(sender.getUniqueId())] = {}
        data[str(sender.getUniqueId())][args[0]] = alias
        save_json_file("aliases", data)
        plugin_header(recipient=sender, name="Chat Alias")
        msg(sender, "&7Chat Alias %s &7==> %s &7successfully created!" % (args[0], alias))
        return True
    else:
        return False

@hook.event("player.AsyncPlayerChatEvent", "high")
def on_player_chat(event):
    playerid = str(event.getPlayer().getUniqueId())
    data = safe_open_json()
    if event.isCancelled():
        return
    if not data[playerid]:
        return
    event.setMessage(multiple_replace(data[playerid], event.getMessage()))

    if (event.getPlayer().hasPermission("essentials.chat.color")):
        event.setMessage(colorify(event.getMessage()))
    if (max_overall_length >= 0) and (len(event.getMessage()) > max_overall_length) and (not event.getPlayer().hasPermission(exceed_overall_length)):
        event.setCancelled(True)
        plugin_header(recipient=event.getPlayer(), name="Chat Alias")
        msg(event.getPlayer(), "&7The message generated was too long and was not sent. :/")

<commit_msg>Fixed a bug in chatalias. Still safe :D<commit_after># Chat Aliasing plugin by Curs3d #
##################################
# Allows users to alias words,
# so that when they send a
# message in chat, it gets
# replaced by their specified
# word. The JSON file for this
# plugin is generated if not
# present. Set values to -1
# for "unlimited" setting.

from helpers import *
import re
from traceback import format_exc as trace

data = None

max_entries = 10
max_alias_length = 35
# Minecraft message limit is 100 so I decided to give a little tolerance (and I added a bit more)
max_overall_length = 100 + max_alias_length

alias_perm = "utils.alias.allowed"
exceed_length = "utils.alias.exceedlimit"
exceed_entries = "utils.alias.exceedlimit"
exceed_overall_length = "utils.alias.exceedlimit"

def safe_open_json():
    global data
    if data is not None:
        return data
    data = open_json_file("aliases")
    if data is None:
        data = {}
    save_json_file("aliases", data)
    return data

def multiple_replace(aliases, text):
    regex = re.compile("|".join(map(re.escape, aliases.keys())))
    return regex.sub(lambda mo: aliases[mo.group(0)], text)

@hook.command("alias",
              usage="/<command> [to_alias] [alias...]",
              desc="Aliases words in chat")
def on_alias_command(sender, cmd, label, args):

    if not is_player(sender):
        msg(sender, "Sorry, non-players cannot alias words")
        return True
    if not sender.hasPermission(alias_perm):
        plugin_header(recipient=sender, name="Chat Alias")
        noperm(sender)
        return True
    if len(args) == 0:
        plugin_header(recipient=sender, name="Chat Alias")
        msg(sender, "&7This is a plugin that allows you to get words" +
            "replaced by other ones automatically!")
        msg(sender, "&7\nCommands:")
        msg(sender, "&e/alias <word> &7removes <word> from your aliases. " +
            "Use &e/alias * &7to remove all aliases.")
        msg(sender, "&e/alias <word> <replacement> &7will change &e<word> " +
            "&7to &e<replacement> &7in chat")
        msg(sender, "&7\nYour Aliases:")
        data = safe_open_json()
        try:
            for alias, value in data[str(sender.getUniqueId())].items():
                msg(sender, "&7%s &7==> %s" % (alias, value))
        except KeyError:
            pass
        return True
    elif len(args) == 1:
        data = safe_open_json()
        if args[0] == "*":
            try:
                del data[str(sender.getUniqueId())]
            except KeyError:
                plugin_header(recipient=sender, name="Chat Alias")
                msg(sender, "&7No alias data to remove!")
                return True
            save_json_file("aliases", data)
            plugin_header(recipient=sender, name="Chat Alias")
            msg(sender, "&cALL &7alias data successfuly removed!")
            return True
        try:
            if data[str(sender.getUniqueId())].pop(args[0], None) is None:
                plugin_header(recipient=sender, name="Chat Alias")
                msg(sender, "&7Could not remove: alias not present!")
                return True
        except KeyError:
            plugin_header(recipient=sender, name="Chat Alias")
            msg(sender, "&7Could not remove: you do not have any aliases!")
            return True
        save_json_file("aliases", data)
        plugin_header(recipient=sender, name="Chat Alias")
        msg(sender, "&7Alias for %s &7successfuly removed" % args[0])
        return True
    elif len(args) >= 2:
        data = safe_open_json()
        alias = " ".join(args[1:])
        try:
            if (len(alias) > max_alias_length) and (max_alias_length >= 0) and (not sender.hasPermission(exceed_length)):
                plugin_header(recipient=sender, name="Chat Alias")
                msg(sender, "&7Please do not alias long words/sentences.")
                return True
            if (len(data[str(sender.getUniqueId())]) >= max_entries) and (max_entries >= 0) and (not sender.hasPermission(exceed_entries)):
                plugin_header(recipient=sender, name="Chat Alias")
                msg(sender, "&7You have reached your alias limit!")
                return True
        except KeyError:
            data[str(sender.getUniqueId())] = {}
        data[str(sender.getUniqueId())][args[0]] = alias
        save_json_file("aliases", data)
        plugin_header(recipient=sender, name="Chat Alias")
        msg(sender, "&7Chat Alias %s &7==> %s &7successfully created!" % (args[0], alias))
        return True
    else:
        return False

@hook.event("player.AsyncPlayerChatEvent", "high")
def on_player_chat(event):
    playerid = str(event.getPlayer().getUniqueId())
    data = safe_open_json()
    if event.isCancelled():
        return
    if not playerid in data:
        return

    event.setMessage(multiple_replace(data[playerid], event.getMessage()))

    if (event.getPlayer().hasPermission("essentials.chat.color")):
        event.setMessage(colorify(event.getMessage()))
    if (max_overall_length >= 0) and (len(event.getMessage()) > max_overall_length) and (not event.getPlayer().hasPermission(exceed_overall_length)):
        event.setCancelled(True)
        plugin_header(recipient=event.getPlayer(), name="Chat Alias")
        msg(event.getPlayer(), "&7The message generated was too long and was not sent. :/")
<|endoftext|>


<commit_before>from buildbot.schedulers.basic import AnyBranchScheduler
from buildbot.schedulers.triggerable import Triggerable
from buildbot.changes.filter import ChangeFilter
from buildbot.process.factory import BuildFactory
from buildbot.steps.master import MasterShellCommand
from buildscripts import steps as buildsteps

project = __opts__['project']

c['schedulers'].append(AnyBranchScheduler(
	name=project,
	change_filter=ChangeFilter(project=project, category='default'),
	builderNames=['{0} source'.format(project)]
))

c["schedulers"].append(Triggerable(
	name="{0} packaging".format(project),
	builderNames=["deb_packaging", "rpm_packaging"]
))

c['builders'].append(dict(
	name='{0} source'.format(project),
	slavenames=['ubuntu1004'],
	factory=BuildFactory(steps=
		buildsteps.svn(__opts__) +
		buildsteps.bump_version(__opts__, setter='cat > cloudstack/version') +
		buildsteps.source_dist(__opts__) +
		buildsteps.trigger_packaging(__opts__) +
		buildsteps.to_repo(__opts__, types=["deb", "rpm"], all_repos=True)
	)
))
<commit_msg>Synced CI instructions from production BuildBot<commit_after>from buildbot.schedulers.basic import AnyBranchScheduler
from buildbot.schedulers.triggerable import Triggerable
from buildbot.changes.filter import ChangeFilter
from buildbot.process.factory import BuildFactory
from buildbot.steps.master import MasterShellCommand
from buildscripts import steps as buildsteps

project = __opts__['project']

c['schedulers'].append(AnyBranchScheduler(
	name=project,
	change_filter=ChangeFilter(project=project, category='default'),
	builderNames=['{0} source'.format(project)]
))

c["schedulers"].append(Triggerable(
	name="{0} packaging".format(project),
	builderNames=["deb_packaging", "rpm_packaging"]
))

c['builders'].append(dict(
	name='{0} source'.format(project),
	slavenames=['ubuntu1004'],
	factory=BuildFactory(steps=
		buildsteps.git(__opts__) +
                buildsteps.bump_version_for_git(__opts__, setter='cat > cloudstack/version') +
		#buildsteps.svn(__opts__) +
		#buildsteps.bump_version(__opts__, setter='cat > cloudstack/version') +
		buildsteps.source_dist(__opts__) +
		buildsteps.trigger_packaging(__opts__) +
		buildsteps.to_repo(__opts__, types=["deb", "rpm"], all_repos=True)
	)
))
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import print_function

from argparse import ArgumentParser
from datetime import (
    datetime,
    timedelta,
    )
import json
import os
import subprocess
import sys

def list_old_juju_containers(hours):
    env = dict(os.environ)
    containers = json.loads(subprocess.check_output([
        'lxc', 'list', '--format', 'json'], env=env))
    now = datetime.now()
    for container in containers:
        name = container['name']
        if not name.startswith('juju-'):
            continue
        # This produces local time.  lxc does not respect TZ=UTC.
        created_at = datetime.strptime(
            container['created_at'][:-6], '%Y-%m-%dT%H:%M:%S')
        age = now - created_at
        if age <= timedelta(hours=hours):
            continue
        yield name, age

def main():
    parser = ArgumentParser('Delete old juju containers')
    parser.add_argument('--dry-run', action='store_true',
                        help='Do not actually delete.')
    parser.add_argument('--hours', type=int, default=1,
                        help='Number of hours a juju container may exist.')
    args = parser.parse_args()
    for container, age in list_old_juju_containers(args.hours):
        print('deleting {} ({} old)'.format(container, age))
        if args.dry_run:
            continue
        subprocess.check_call(('lxc', 'delete', '--verbose', '--force',
                               container))

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Use dateutil to calculate age of container.<commit_after>#!/usr/bin/env python
from __future__ import print_function

from argparse import ArgumentParser
from datetime import (
    datetime,
    timedelta,
    )
import json
import os
import subprocess
import sys

from dateutil import (
    parser as date_parser,
    tz,
    )

def list_old_juju_containers(hours):
    env = dict(os.environ)
    containers = json.loads(subprocess.check_output([
        'lxc', 'list', '--format', 'json'], env=env))
    now = datetime.now(tz.gettz('UTC'))
    for container in containers:
        name = container['name']
        if not name.startswith('juju-'):
            continue
        created_at = date_parser.parse(container['created_at'])
        age = now - created_at
        if age <= timedelta(hours=hours):
            continue
        yield name, age

def main():
    parser = ArgumentParser('Delete old juju containers')
    parser.add_argument('--dry-run', action='store_true',
                        help='Do not actually delete.')
    parser.add_argument('--hours', type=int, default=1,
                        help='Number of hours a juju container may exist.')
    args = parser.parse_args()
    for container, age in list_old_juju_containers(args.hours):
        print('deleting {} ({} old)'.format(container, age))
        if args.dry_run:
            continue
        subprocess.check_call(('lxc', 'delete', '--verbose', '--force',
                               container))

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>from setuptools import setup

version = {}
with open('popper/__init__.py') as f:
    exec(f.read(), version)

setup(
    name='popper',
    version=version['__version__'],
    author='The Popper Development Team',
    author_email='ivo@cs.ucsc.edu',
    url='http://falsifiable.us',
    description='Popper CLI tool to generate reproducible papers.',
    packages=['popper', 'popper.commands'],
    include_package_data=True,
    install_requires=[
        'python-vagrant',
        'GitPython',
        'spython',
        'click',
        'pyhcl==0.4.0',
        'pyyaml',
        'docker'
    ],
    extras_require={
        'dev': ['testfixtures']
    },
    entry_points='''
        [console_scripts]
        popper=popper.cli:cli
    ''',
    project_urls={
        'Documentation': 'https://popper.readthedocs.io',
        'Say Thanks!': 'http://gitter.im/falsifiable-us/popper',
        'Source': 'https://github.com/systemslab/popper/',
        'Tracker': 'https://github.com/systemslab/popper/',
    },
    keywords='popper reproducibility automation continuous-integration',
    license='MIT',
)
<commit_msg>Pin dependencies to specific versions<commit_after>from setuptools import setup

version = {}
with open('popper/__init__.py') as f:
    exec(f.read(), version)

setup(
    name='popper',
    version=version['__version__'],
    author='The Popper Development Team',
    author_email='ivo@cs.ucsc.edu',
    url='http://falsifiable.us',
    description='Popper CLI tool to generate reproducible papers.',
    packages=['popper', 'popper.commands'],
    include_package_data=True,
    install_requires=[
        'click==7.1.2',
        'docker==4.2.0',
        'GitPython==3.1.0',
        'pyhcl==0.4.0',
        'pyyaml==5.3.1',
        'python-vagrant==0.5.15',
        'spython==0.0.79',
    ],
    extras_require={
        'dev': ['testfixtures']
    },
    entry_points='''
        [console_scripts]
        popper=popper.cli:cli
    ''',
    project_urls={
        'Documentation': 'https://popper.readthedocs.io',
        'Say Thanks!': 'http://gitter.im/falsifiable-us/popper',
        'Source': 'https://github.com/systemslab/popper/',
        'Tracker': 'https://github.com/systemslab/popper/',
    },
    keywords='popper reproducibility automation continuous-integration',
    license='MIT',
)
<|endoftext|>