<commit_before>import sys
import signals
import logging
from pyethereum import rlp
from pyethereum.utils import big_endian_to_int as idec
from pyethereum.utils import int_to_big_endian4 as ienc4
from pyethereum.utils import recursive_int_to_big_endian
from pyethereum.utils import sha3
from pyethereum import dispatch
from . import __version__

logger = logging.getLogger(__name__)

def lrlp_decode(data):
    "always return a list"
    d = rlp.decode(data)
    if isinstance(d, str):
        d = [d]
    return d

def load_packet(packet):
    return Packeter.load_packet(packet)

class Packeter(object):
    """
    Translates between the network and the local data
    https://github.com/ethereum/wiki/wiki/%5BEnglish%5D-Wire-Protocol
    https://github.com/ethereum/cpp-ethereum/wiki/%C3%90%CE%9EVP2P-Networking

    """
    NETWORK_PROTOCOL_VERSION = 0
    ETHEREUM_PROTOCOL_VERSION = 36  # IF CHANGED, DO: git tag 0.6.<ETHEREUM_PROTOCOL_VERSION>
    CLIENT_VERSION = 'Ethereum(py)/%s/%s' % (sys.platform, __version__)
    #the node s Unique Identifier and is the 512-bit hash that serves to identify the node.
    NODE_ID = sha3('')  # set in config
    NETWORK_ID = 0
    SYNCHRONIZATION_TOKEN = 0x22400891
    CAPABILITIES = ['eth']  # + ['shh']  ethereum protocol  whisper protocol

    cmd_map = dict(((0x00, 'Hello'),
                   (0x01, 'Disconnect'),
                   (0x02, 'Ping'),
                   (0x03, 'Pong'),
                   (0x04, 'GetPeers'),
                   (0x05, 'Peers'),
                   (0x10, 'Status'),
                   (0x11, 'GetTransactions'),
                   (0x12, 'Transactions'),
                   (0x13, 'GetBlockHashes'),
                   (0x14, 'BlockHashes'),
                   (0x15, 'GetBlocks'),
                   (0x16, 'Blocks'),
                   ))
    cmd_map_by_name = dict((v, k) for k, v in cmd_map.items())

    disconnect_reasons_map = dict((
        ('Disconnect requested', 0x00),
        ('TCP sub-system error', 0x01),
        ('Bad protocol', 0x02),
        ('Useless peer', 0x03),
        ('Too many peers', 0x04),
        ('Already connected', 0x05),
        ('Wrong genesis block', 0x06),
        ('Incompatible network protocols', 0x07),
        ('Client quitting', 0x08)))
    disconnect_reasons_map_by_id = \
        dict((v, k) for k, v in disconnect_reasons_map.items())

    def __init__(self):
        pass

    def configure(self, config):
        self.config = config
        self.CLIENT_VERSION = self.config.get('network', 'client_version') \
            or self.CLIENT_VERSION
        self.NODE_ID = self.config.get('network', 'node_id')

    @classmethod
    def packet_size(cls, packet):
        return idec(packet[4:8]) + 8

    @classmethod
    def packet_cmd(cls, packet):
        try:
            v = idec(rlp.descend(packet[8:200], 0))
        except rlp.DecodingError:
            v = -1
        return Packeter.cmd_map.get(v, 'invalid')

    @classmethod
    def load_packet(cls, packet):
        '''
        Though TCP provides a connection-oriented medium, Ethereum nodes
        communicate in terms of packets. These packets are formed as a 4-byte
        synchronisation token (0x22400891), a 4-byte "payload size", to be
        interpreted as a big-endian integer and finally an N-byte
        RLP-serialised data structure, where N is the aforementioned
        "payload size". To be clear, the payload size specifies the number of
        bytes in the packet ''following'' the first 8.

        :return: (success, result), where result should be None when fail,
        and (header, payload_len, cmd, data) when success
        '''
        header = idec(packet[:4])
        if header != cls.SYNCHRONIZATION_TOKEN:
            return False, 'check header failed, skipping message,'\
                'sync token was hex: %s' % hex(header)

        try:
            payload_len = idec(packet[4:8])
        except Exception as e:
            return False, str(e)

        if len(packet) < payload_len + 8:
            return False, 'Packet is broken'

        try:
            payload = lrlp_decode(packet[8:8 + payload_len])
        except Exception as e:
            return False, str(e)

        #logger.debug('load packet, cmd:%d %r', idec(payload[0]), Packeter.cmd_map.get(idec(payload[0]),'unknown'))
        if (not len(payload)) or (idec(payload[0]) not in cls.cmd_map):
            return False, 'check cmd %r failed' % idec(payload[0])

        cmd = Packeter.cmd_map.get(idec(payload[0]))
        remain = packet[8 + payload_len:]
        return True, (header, payload_len, cmd, payload[1:], remain)

    def load_cmd(self, packet):
        success, res = self.load_packet(packet)
        if not success:
            raise Exception(res)
        _, _, cmd, data, remain = res
        return cmd, data, remain

    @classmethod
    def dump_packet(cls, data):
        """
        4-byte synchronisation token, (0x22400891),
        a 4-byte "payload size", to be interpreted as a big-endian integer
        an N-byte RLP-serialised data structure
        """
        payload = rlp.encode(recursive_int_to_big_endian(data))

        packet = ienc4(cls.SYNCHRONIZATION_TOKEN)
        packet += ienc4(len(payload))
        packet += payload
        return packet

    def dump_Hello(self):
        """
        0x01 Hello: [0x01: P, protocolVersion: P, clientVersion: B, [cap0: B, cap1: B, ...], listenPort: P, id: B_64]

        protocolVersion: The underlying network protocol. 0
        clientVersion: The underlying client. A user-readable string.
        capN: A peer-network capability code, readable ASCII and 3 letters. Currently only "eth" and "shh" are known.
        listenPort: The port on which the peer is listening for an incoming connection.
        id: The identity and public key of the peer.
        """
        data = [self.cmd_map_by_name['Hello'],
                self.NETWORK_PROTOCOL_VERSION,
                self.CLIENT_VERSION,
                self.CAPABILITIES,
                self.config.getint('network', 'listen_port'),
                self.NODE_ID
                ]
        return self.dump_packet(data)

    def dump_Ping(self):
        data = [self.cmd_map_by_name['Ping']]
        return self.dump_packet(data)

    def dump_Pong(self):
        data = [self.cmd_map_by_name['Pong']]
        return self.dump_packet(data)

    def dump_Disconnect(self, reason=None):
        data = [self.cmd_map_by_name['Disconnect']]
        if reason:
            data.append(self.disconnect_reasons_map[reason])
        return self.dump_packet(data)

    def dump_GetPeers(self):
        data = [self.cmd_map_by_name['GetPeers']]
        return self.dump_packet(data)

    def dump_Peers(self, peers):
        '''
        :param peers: a sequence of (ip, port, pid)
        :return: None if no peers
        '''
        data = [self.cmd_map_by_name['Peers']]
        for ip, port, pid in peers:
            assert ip.count('.') == 3
            ip = ''.join(chr(int(x)) for x in ip.split('.'))
            data.append([ip, port, pid])
        return self.dump_packet(data)

    def dump_Status(self, total_difficulty, head_hash, genesis_hash):
        """
        0x10 Status: [0x10: P, protocolVersion: P, networkID: P, totalDifficulty: P, latestHash: B_32, genesisHash: B_32]

        protocolVersion: The version of the Ethereum protocol this peer implements. 30 at present.
        networkID: The network version of Ethereum for this peer. 0 for the official testnet.
        totalDifficulty: Total Difficulty of the best chain. Integer, as found in block header.
        latestHash: The hash of the block with the highest validated total difficulty.
        GenesisHash: The hash of the Genesis block.
        """
        data = [self.cmd_map_by_name['Status'],
                self.ETHEREUM_PROTOCOL_VERSION,
                self.NETWORK_ID,
                total_difficulty,  # chain head total difficulty,
                head_hash,  # chain head hash
                genesis_hash  # genesis hash
                ]
        return self.dump_packet(data)

    def dump_Transactions(self, transactions):
        data = [self.cmd_map_by_name['Transactions']] + transactions
        return self.dump_packet(data)

    def dump_GetTransactions(self):
        """
        [0x12, [nonce, receiving_address, value, ... ], ... ]
        Specify (a) transaction(s) that the peer should make sure is included
        on its transaction queue. The items in the list (following the first
        item 0x12) are transactions in the format described in the main
        Ethereum specification.
        """
        data = [self.cmd_map_by_name['GetTransactions']]
        return self.dump_packet(data)

    def dump_Blocks(self, blocks):
        blocks_as_lists = [rlp.decode(b.serialize()) for b in blocks]
        # FIXME, can we have a method to append rlp encoded data
        data = [self.cmd_map_by_name['Blocks']] + blocks_as_lists
        return self.dump_packet(data)

    def dump_GetBlockHashes(self, block_hash, max_blocks):
        """
        [0x17, [ hash : B_32, maxBlocks: P ]]
        Requests a BlockHashes message of at most maxBlocks entries, of block hashes from
        the blockchain, starting at the parent of block hash. Does not require the peer
        to give maxBlocks hashes - they could give somewhat fewer.
        """
        data = [self.cmd_map_by_name['GetBlockHashes'], block_hash, max_blocks]
        return self.dump_packet(data)

    def dump_BlockHashes(self, block_hashes):
        """
        [0x18, [ hash_0: B_32, hash_1: B_32, .... ]]
        Gives a series of hashes of blocks (each the child of the next). This implies that
        the blocks are ordered from youngest to oldest.
        """
        data = [self.cmd_map_by_name['BlockHashes']] + block_hashes
        return self.dump_packet(data)

    def dump_GetBlocks(self, block_hashes):
        """
        [0x19,[ hash_0: B_32, hash_1: B_32, .... ]]
        Requests a Blocks message detailing a number of blocks to be sent, each referred to
        by a hash. Note: Don't expect that the peer necessarily give you all these blocks
        in a single message - you might have to re-request them.
        """
        data = [self.cmd_map_by_name['GetBlocks']] + block_hashes
        return self.dump_packet(data)

packeter = Packeter()

@dispatch.receiver(signals.config_ready)
def config_packeter(sender, config, **kwargs):
    packeter.configure(config)
<commit_msg>fake adaptive messages for eth only<commit_after>import sys
import signals
import logging
from pyethereum import rlp
from pyethereum.utils import big_endian_to_int as idec
from pyethereum.utils import int_to_big_endian4 as ienc4
from pyethereum.utils import recursive_int_to_big_endian
from pyethereum.utils import sha3
from pyethereum import dispatch
from . import __version__

logger = logging.getLogger(__name__)

def lrlp_decode(data):
    "always return a list"
    d = rlp.decode(data)
    if isinstance(d, str):
        d = [d]
    return d

def load_packet(packet):
    return Packeter.load_packet(packet)

class Packeter(object):

    """
    Translates between the network and the local data
    https://github.com/ethereum/wiki/wiki/%5BEnglish%5D-Wire-Protocol
    https://github.com/ethereum/cpp-ethereum/wiki/%C3%90%CE%9EVP2P-Networking

    """
    NETWORK_PROTOCOL_VERSION = 2
    # IF CHANGED, DO: git tag 0.6.<ETHEREUM_PROTOCOL_VERSION>
    ETHEREUM_PROTOCOL_VERSION = 38
    CLIENT_VERSION = 'Ethereum(py)/%s/%s' % (sys.platform, __version__)
    # the node s Unique Identifier and is the 512-bit hash that serves to
    # identify the node.
    NODE_ID = sha3('')  # set in config
    NETWORK_ID = 0
    SYNCHRONIZATION_TOKEN = 0x22400891

    p2p_cmd_map = dict((
        (0x00, 'Hello'),
        (0x01, 'Disconnect'),
        (0x02, 'Ping'),
        (0x03, 'Pong'),
        (0x04, 'GetPeers'),
        (0x05, 'Peers')
    ))

    eth_cmd_map = dict((
        (0x00, 'Status'),
        (0x02, 'Transactions'),
        (0x03, 'GetBlockHashes'),
        (0x04, 'BlockHashes'),
        (0x05, 'GetBlocks'),
        (0x06, 'Blocks'),
        (0x07, 'NewBlock')
    ))

    # FAKE ADAPTIVE MESSAGE IDs (eth,eth)
    cmd_map = dict(p2p_cmd_map)
    offset = 11 + max(p2p_cmd_map.keys())
    for _id, name in eth_cmd_map.items():
        cmd_map[offset + _id] = name

    cmd_map_by_name = dict((v, k) for k, v in cmd_map.items())

    # for now we only have eth capabilities.
    # if we add, shh we also need to add Adaptive Message IDs
    # + ['shh', 2]  ethereum protocol  whisper protocol
    CAPABILITIES = [('eth', ETHEREUM_PROTOCOL_VERSION)]

    disconnect_reasons_map = dict((
        ('Disconnect requested', 0x00),
        ('TCP sub-system error', 0x01),
        ('Bad protocol', 0x02),
        ('Useless peer', 0x03),
        ('Too many peers', 0x04),
        ('Already connected', 0x05),
        ('Wrong genesis block', 0x06),
        ('Incompatible network protocols', 0x07),
        ('Client quitting', 0x08)))

    disconnect_reasons_map_by_id = \
        dict((v, k) for k, v in disconnect_reasons_map.items())

    def __init__(self):
        pass

    def configure(self, config):
        self.config = config
        self.CLIENT_VERSION = self.config.get('network', 'client_version') \
            or self.CLIENT_VERSION
        self.NODE_ID = self.config.get('network', 'node_id')

    @classmethod
    def packet_size(cls, packet):
        return idec(packet[4:8]) + 8

    @classmethod
    def packet_cmd(cls, packet):
        try:
            v = idec(rlp.descend(packet[8:200], 0))
        except rlp.DecodingError:
            v = -1
        return Packeter.cmd_map.get(v, 'invalid')

    @classmethod
    def load_packet(cls, packet):
        '''
        Though TCP provides a connection-oriented medium, Ethereum nodes
        communicate in terms of packets. These packets are formed as a 4-byte
        synchronisation token (0x22400891), a 4-byte "payload size", to be
        interpreted as a big-endian integer and finally an N-byte
        RLP-serialised data structure, where N is the aforementioned
        "payload size". To be clear, the payload size specifies the number of
        bytes in the packet ''following'' the first 8.

        :return: (success, result), where result should be None when fail,
        and (header, payload_len, cmd, data) when success
        '''
        header = idec(packet[:4])
        if header != cls.SYNCHRONIZATION_TOKEN:
            return False, 'check header failed, skipping message,'\
                'sync token was hex: %s' % hex(header)

        try:
            payload_len = idec(packet[4:8])
        except Exception as e:
            return False, str(e)

        if len(packet) < payload_len + 8:
            return False, 'Packet is broken'

        try:
            payload = lrlp_decode(packet[8:8 + payload_len])
        except Exception as e:
            return False, str(e)

        #logger.debug('load packet, cmd:%d %r', idec(payload[0]), Packeter.cmd_map.get(idec(payload[0]),'unknown'))
        if (not len(payload)) or (idec(payload[0]) not in cls.cmd_map):
            return False, 'check cmd %r failed' % idec(payload[0])

        cmd = Packeter.cmd_map.get(idec(payload[0]))
        remain = packet[8 + payload_len:]
        return True, (header, payload_len, cmd, payload[1:], remain)

    def load_cmd(self, packet):
        success, res = self.load_packet(packet)
        if not success:
            raise Exception(res)
        _, _, cmd, data, remain = res
        return cmd, data, remain

    @classmethod
    def dump_packet(cls, data):
        """
        4-byte synchronisation token, (0x22400891),
        a 4-byte "payload size", to be interpreted as a big-endian integer
        an N-byte RLP-serialised data structure
        """
        payload = rlp.encode(recursive_int_to_big_endian(data))

        packet = ienc4(cls.SYNCHRONIZATION_TOKEN)
        packet += ienc4(len(payload))
        packet += payload
        return packet

    def dump_Hello(self):
        """
        0x01 Hello: [0x01: P, protocolVersion: P, clientVersion: B, [cap0: B, cap1: B, ...], listenPort: P, id: B_64]

        protocolVersion: The underlying network protocol. 0
        clientVersion: The underlying client. A user-readable string.
        capN: A peer-network capability code, readable ASCII and 3 letters. Currently only "eth" and "shh" are known.
        listenPort: The port on which the peer is listening for an incoming connection.
        id: The identity and public key of the peer.
        """
        data = [self.cmd_map_by_name['Hello'],
                self.NETWORK_PROTOCOL_VERSION,
                self.CLIENT_VERSION,
                self.CAPABILITIES,
                self.config.getint('network', 'listen_port'),
                self.NODE_ID
                ]
        return self.dump_packet(data)

    def dump_Ping(self):
        data = [self.cmd_map_by_name['Ping']]
        return self.dump_packet(data)

    def dump_Pong(self):
        data = [self.cmd_map_by_name['Pong']]
        return self.dump_packet(data)

    def dump_Disconnect(self, reason=None):
        data = [self.cmd_map_by_name['Disconnect']]
        if reason:
            data.append(self.disconnect_reasons_map[reason])
        return self.dump_packet(data)

    def dump_GetPeers(self):
        data = [self.cmd_map_by_name['GetPeers']]
        return self.dump_packet(data)

    def dump_Peers(self, peers):
        '''
        :param peers: a sequence of (ip, port, pid)
        :return: None if no peers
        '''
        data = [self.cmd_map_by_name['Peers']]
        for ip, port, pid in peers:
            assert ip.count('.') == 3
            ip = ''.join(chr(int(x)) for x in ip.split('.'))
            data.append([ip, port, pid])
        return self.dump_packet(data)

    def dump_Status(self, total_difficulty, head_hash, genesis_hash):
        """
        0x10 Status: [0x10: P, protocolVersion: P, networkID: P, totalDifficulty: P, latestHash: B_32, genesisHash: B_32]

        protocolVersion: The version of the Ethereum protocol this peer implements. 30 at present.
        networkID: The network version of Ethereum for this peer. 0 for the official testnet.
        totalDifficulty: Total Difficulty of the best chain. Integer, as found in block header.
        latestHash: The hash of the block with the highest validated total difficulty.
        GenesisHash: The hash of the Genesis block.
        """
        data = [self.cmd_map_by_name['Status'],
                self.ETHEREUM_PROTOCOL_VERSION,
                self.NETWORK_ID,
                total_difficulty,  # chain head total difficulty,
                head_hash,  # chain head hash
                genesis_hash  # genesis hash
                ]
        return self.dump_packet(data)

    def dump_Transactions(self, transactions):
        data = [self.cmd_map_by_name['Transactions']] + transactions
        return self.dump_packet(data)

    def dump_GetTransactions(self):
        """
        [0x12, [nonce, receiving_address, value, ... ], ... ]
        Specify (a) transaction(s) that the peer should make sure is included
        on its transaction queue. The items in the list (following the first
        item 0x12) are transactions in the format described in the main
        Ethereum specification.
        """
        data = [self.cmd_map_by_name['GetTransactions']]
        return self.dump_packet(data)

    def dump_Blocks(self, blocks):
        blocks_as_lists = [rlp.decode(b.serialize()) for b in blocks]
        # FIXME, can we have a method to append rlp encoded data
        data = [self.cmd_map_by_name['Blocks']] + blocks_as_lists
        return self.dump_packet(data)

    def dump_GetBlockHashes(self, block_hash, max_blocks):
        """
        [0x17, [ hash : B_32, maxBlocks: P ]]
        Requests a BlockHashes message of at most maxBlocks entries, of block hashes from
        the blockchain, starting at the parent of block hash. Does not require the peer
        to give maxBlocks hashes - they could give somewhat fewer.
        """
        data = [self.cmd_map_by_name['GetBlockHashes'], block_hash, max_blocks]
        return self.dump_packet(data)

    def dump_BlockHashes(self, block_hashes):
        """
        [0x18, [ hash_0: B_32, hash_1: B_32, .... ]]
        Gives a series of hashes of blocks (each the child of the next). This implies that
        the blocks are ordered from youngest to oldest.
        """
        data = [self.cmd_map_by_name['BlockHashes']] + block_hashes
        return self.dump_packet(data)

    def dump_GetBlocks(self, block_hashes):
        """
        [0x19,[ hash_0: B_32, hash_1: B_32, .... ]]
        Requests a Blocks message detailing a number of blocks to be sent, each referred to
        by a hash. Note: Don't expect that the peer necessarily give you all these blocks
        in a single message - you might have to re-request them.
        """
        data = [self.cmd_map_by_name['GetBlocks']] + block_hashes
        return self.dump_packet(data)

packeter = Packeter()

@dispatch.receiver(signals.config_ready)
def config_packeter(sender, config, **kwargs):
    packeter.configure(config)
<|endoftext|>


<commit_before># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import re
import ipaddress
from warnings import warn
from sqlalchemy.orm import validates

from pyfarm.flaskapp import db
from pyfarm.warning import NetworkWarning
from pyfarm.ext.config.core.loader import Loader
from pyfarm.ext.config.enum import AgentState
from pyfarm.ext.utility import isLocalIPv4Address
from pyfarm.models.mixins import StateValidationMixin, RandIdMixin
from pyfarm.models.constants import (
    DBCFG, TABLE_AGENT, TABLE_AGENT_TAGS, TABLE_AGENT_SOFTWARE)

REGEX_HOSTNAME = re.compile("^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*"
                            "[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9]"
                            "[A-Za-z0-9\-]*[A-Za-z0-9])$")

class AgentTags(db.Model):
    """Table model used to store tags for agents"""
    __tablename__ = TABLE_AGENT_TAGS
    _agentid = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_AGENT),
                         primary_key=True)
    tag = db.Column(db.String)

class AgentSoftware(db.Model):
    """Table model used to store tags for agents"""
    __tablename__ = TABLE_AGENT_SOFTWARE
    _agentid = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_AGENT),
                         primary_key=True)
    software = db.Column(db.String)

class Agent(db.Model, RandIdMixin, StateValidationMixin):
    """
    Stores information about an agent include its network address,
    state, allocation configuration, etc.
    """
    __tablename__ = TABLE_AGENT
    STATE_ENUM = AgentState()
    STATE_DEFAULT = STATE_ENUM.ONLINE

    # NOTE: any columns where are null will generally be filled in
    # by the client on first connect

    # host state information
    enabled = db.Column(db.Boolean, default=True)
    state = db.Column(db.Integer, default=STATE_DEFAULT, nullable=False)
    hostname = db.Column(db.String, nullable=False)
    ip = db.Column(db.String(15), nullable=False)
    subnet = db.Column(db.String(15), nullable=False)
    ram = db.Column(db.Integer)
    cpus = db.Column(db.Integer)
    port = db.Column(db.Integer)

    # Max allocation of the two primary resources which `1.0` is 100%
    # allocation.  For `cpu_allocation` 100% allocation typically means
    # one task per cpu.
    ram_allocation = db.Column(db.Float,
                               default=DBCFG.get("agent.ram_allocation", .8))
    cpu_allocation = db.Column(db.Float,
                               default=DBCFG.get("agent.cpu_allocation", 1.0))

    # relationships
    tasks = db.relationship("Task", backref="agent", lazy="dynamic")
    tags = db.relationship("AgentTags", backref="agent", lazy="dynamic")
    software = db.relation("AgentSoftware", backref="agent", lazy="dynamic")

    @validates("hostname")
    def validate_hostname(self, key, value):
        # ensure hostname does not contain characters we can't use
        if not REGEX_HOSTNAME.match(value):
            raise ValueError("%s is not valid for %s" % (value, key))

        return value

    @validates("ip", "subnet")
    def validate_address(self, key, value):
        try:
            if isLocalIPv4Address(value):
                warn("non-public address %s provided" % value, NetworkWarning)

        except ipaddress.AddressValueError, e:
            raise ipaddress.AddressValueError(
                "%s is not a valid ip: %s" % (value, e))

        return value

    @validates("ram", "cpus", "port")
    def validate_resource(self, key, value):
        if value is None:
            return value

        min_value = DBCFG.get("agent.min_%s" % key)
        max_value = DBCFG.get("agent.max_%s" % key)

        # quick sanity check of the incoming config
        assert isinstance(min_value, int), "db.min_%s must be an integer" % key
        assert isinstance(max_value, int), "db.max_%s must be an integer" % key
        assert min_value >= 1, "db.min_%s must be > 0" % key
        assert max_value >= 1, "db.max_%s must be > 0" % key

        # check the provided input
        if min_value > value or value > max_value:
            msg = "value for `%s` must be between " % key
            msg += "%s and %s" % (min_value, max_value)
            raise ValueError(msg)

        return value

<commit_msg>don't need the warning<commit_after># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import re
import ipaddress
from sqlalchemy.orm import validates

from pyfarm.flaskapp import db
from pyfarm.ext.config.core.loader import Loader
from pyfarm.ext.config.enum import AgentState
from pyfarm.ext.utility import isLocalIPv4Address
from pyfarm.models.mixins import StateValidationMixin, RandIdMixin
from pyfarm.models.constants import (
    DBCFG, TABLE_AGENT, TABLE_AGENT_TAGS, TABLE_AGENT_SOFTWARE)

REGEX_HOSTNAME = re.compile("^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*"
                            "[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9]"
                            "[A-Za-z0-9\-]*[A-Za-z0-9])$")

class AgentTags(db.Model):
    """Table model used to store tags for agents"""
    __tablename__ = TABLE_AGENT_TAGS
    _agentid = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_AGENT),
                         primary_key=True)
    tag = db.Column(db.String)

class AgentSoftware(db.Model):
    """Table model used to store tags for agents"""
    __tablename__ = TABLE_AGENT_SOFTWARE
    _agentid = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_AGENT),
                         primary_key=True)
    software = db.Column(db.String)

class Agent(db.Model, RandIdMixin, StateValidationMixin):
    """
    Stores information about an agent include its network address,
    state, allocation configuration, etc.
    """
    __tablename__ = TABLE_AGENT
    STATE_ENUM = AgentState()
    STATE_DEFAULT = STATE_ENUM.ONLINE

    # NOTE: any columns where are null will generally be filled in
    # by the client on first connect

    # host state information
    enabled = db.Column(db.Boolean, default=True)
    state = db.Column(db.Integer, default=STATE_DEFAULT, nullable=False)
    hostname = db.Column(db.String, nullable=False)
    ip = db.Column(db.String(15), nullable=False)
    subnet = db.Column(db.String(15), nullable=False)
    ram = db.Column(db.Integer)
    cpus = db.Column(db.Integer)
    port = db.Column(db.Integer)

    # Max allocation of the two primary resources which `1.0` is 100%
    # allocation.  For `cpu_allocation` 100% allocation typically means
    # one task per cpu.
    ram_allocation = db.Column(db.Float,
                               default=DBCFG.get("agent.ram_allocation", .8))
    cpu_allocation = db.Column(db.Float,
                               default=DBCFG.get("agent.cpu_allocation", 1.0))

    # relationships
    tasks = db.relationship("Task", backref="agent", lazy="dynamic")
    tags = db.relationship("AgentTags", backref="agent", lazy="dynamic")
    software = db.relation("AgentSoftware", backref="agent", lazy="dynamic")

    @validates("hostname")
    def validate_hostname(self, key, value):
        # ensure hostname does not contain characters we can't use
        if not REGEX_HOSTNAME.match(value):
            raise ValueError("%s is not valid for %s" % (value, key))

        return value

    @validates("ip", "subnet")
    def validate_address(self, key, value):
        try:
            isLocalIPv4Address(value)

        except ipaddress.AddressValueError, e:
            raise ipaddress.AddressValueError(
                "%s is not a valid ip: %s" % (value, e))

        return value

    @validates("ram", "cpus", "port")
    def validate_resource(self, key, value):
        if value is None:
            return value

        min_value = DBCFG.get("agent.min_%s" % key)
        max_value = DBCFG.get("agent.max_%s" % key)

        # quick sanity check of the incoming config
        assert isinstance(min_value, int), "db.min_%s must be an integer" % key
        assert isinstance(max_value, int), "db.max_%s must be an integer" % key
        assert min_value >= 1, "db.min_%s must be > 0" % key
        assert max_value >= 1, "db.max_%s must be > 0" % key

        # check the provided input
        if min_value > value or value > max_value:
            msg = "value for `%s` must be between " % key
            msg += "%s and %s" % (min_value, max_value)
            raise ValueError(msg)

        return value

<|endoftext|>


<commit_before>#
# Copyright 2015 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import division
from collections import defaultdict

import pandas as pd
import numpy as np

def extract_round_trips(transactions):
    """
    Group transactions into "round trips." A round trip is started when a new
    long or short position is opened and is only completed when the number
    of shares in that position returns to or crosses zero.
    Computes pnl for each round trip.

    For example, the following transactions would constitute one round trip:
    index                  amount   price    symbol
    2004-01-09 12:18:01    186      324.12   'AAPL'
    2004-01-09 15:12:53    -10      344.54   'AAPL'
    2004-01-13 14:41:23    24       320.21   'AAPL'
    2004-01-30 10:23:34    -200     340.43   'AAPL'

    Parameters
    ----------
    transactions : pd.DataFrame
        Prices and amounts of executed trades. One row per trade.
        - See full explanation in tears.create_full_tear_sheet

    Returns
    -------
    round_trips : pd.DataFrame
        DataFrame with one row per round trip.
    """
    # Transactions that cross zero must be split into separate
    # long and short transactions that end/start on zero.
    transactions_split = split_trades(transactions)

    transactions_split['txn_dollars'] =  \
        -transactions_split['amount'] * transactions_split['price']

    round_trips = defaultdict(list)

    for sym, trans_sym in transactions_split.groupby('symbol'):
        trans_sym = trans_sym.sort_index()
        amount_cumsum = trans_sym.amount.cumsum()
        # Find indicies where the position amount returns to zero.
        closed_idx = np.where(amount_cumsum == 0)[0] + 1
        # Identify the first trade as the beginning of a round trip.
        closed_idx = np.insert(closed_idx, 0, 0)

        for trade_start, trade_end in zip(closed_idx, closed_idx[1:]):
            txn = trans_sym.iloc[trade_start:trade_end]

            if len(txn) == 0:
                continue

            assert txn.amount.sum() == 0
            long_trade = txn.amount.iloc[0] > 0
            pnl = txn.txn_dollars.sum()
            round_trips['symbol'].append(sym)
            round_trips['pnl'].append(pnl)
            round_trips['duration'].append(txn.index[-1] - txn.index[0])
            round_trips['long'].append(long_trade)
            round_trips['open_dt'].append(txn.index[0])
            round_trips['close_dt'].append(txn.index[-1])

            # Investing txns push the position amount farther from zero.
            # invested is always a positive value. Returned - Invested = PnL.
            if long_trade:
                invested = -txn.query('txn_dollars < 0').txn_dollars.sum()
            else:
                invested = txn.query('txn_dollars > 0').txn_dollars.sum()

            if invested == 0:
                round_trips['returns'].append(0)
            else:
                round_trips['returns'].append(pnl / invested)

    if len(round_trips) == 0:
        return pd.DataFrame([])

    round_trips = pd.DataFrame(round_trips)
    round_trips = round_trips[['open_dt', 'close_dt', 'duration',
                               'pnl', 'returns', 'long', 'symbol']]

    return round_trips

def split_trades(transactions):
    """
    Splits transactions that cause total position amount to cross zero.
    In other words, separates of the closing of one short/long position
    with the opening of a new long/short position.

    For example, the second transaction in this transactions DataFrame
    would be divided as shown in the second DataFrame:
    index                  amount   price    symbol
    2004-01-09 12:18:01    180      324.12   'AAPL'
    2004-01-09 15:12:53    -200     344.54   'AAPL'

    index                  amount   price    symbol
    2004-01-09 12:18:01    180      324.12   'AAPL'
    2004-01-09 15:12:53    -180     344.54   'AAPL'
    2004-01-09 15:12:54    -20      344.54   'AAPL'

    Parameters
    ----------
    transactions : pd.DataFrame
        Prices and amounts of executed trades. One row per trade.
        - See full explanation in tears.create_full_tear_sheet

    Returns
    -------
    transactions_split : pd.DataFrame
        Prices and amounts of executed trades. Trades that cause
        total position amount to cross zero are divided.
    """

    trans_split = []

    for sym, trans_sym in transactions.groupby('symbol'):
        trans_sym = trans_sym.sort_index()

        while True:
            cum_amount = trans_sym.amount.cumsum()
            # find the indicies where position amount crosses zero
            sign_flip = np.where(np.abs(np.diff(np.sign(cum_amount))) == 2)[0]

            if len(sign_flip) == 0:
                break  # all sign flips are converted

            sign_flip = sign_flip[0] + 2

            txn = trans_sym.iloc[:sign_flip]

            left_over_txn_amount = txn.amount.sum()
            assert left_over_txn_amount != 0

            split_txn_1 = txn.iloc[[-1]].copy()
            split_txn_2 = txn.iloc[[-1]].copy()

            split_txn_1['amount'] -= left_over_txn_amount
            split_txn_2['amount'] = left_over_txn_amount

            # Delay 2nd trade by a second to avoid overlapping indices
            split_txn_2.index += pd.Timedelta(seconds=1)

            assert split_txn_1.amount.iloc[0] + \
                split_txn_2.amount.iloc[0] == txn.iloc[-1].amount
            assert trans_sym.iloc[:sign_flip - 1].amount.sum() + \
                split_txn_1.amount.iloc[0] == 0

            # Recreate transactions so far with split transaction
            trans_sym = pd.concat([trans_sym.iloc[:sign_flip - 1],
                                   split_txn_1,
                                   split_txn_2,
                                   trans_sym.iloc[sign_flip:]])

        assert np.all(np.abs(np.diff(np.sign(trans_sym.amount.cumsum()))) != 2)
        trans_split.append(trans_sym)

    transactions_split = pd.concat(trans_split)

    return transactions_split

def add_closing_transactions(positions, transactions):
    """
    Appends transactions that close out all positions at the end of
    the timespan covered by positions data. Utilizes pricing information
    in the positions DataFrame to determine closing price.

    Parameters
    ----------
    positions : pd.DataFrame
        The positions that the strategy takes over time.
    transactions : pd.DataFrame
        Prices and amounts of executed trades. One row per trade.
        - See full explanation in tears.create_full_tear_sheet

    Returns
    -------
    closed_txns : pd.DataFrame
        Transactions with closing transactions appended.
    """

    closed_txns = transactions.copy()

    pos_at_end = positions.drop('cash', axis=1).iloc[-1]
    open_pos = pos_at_end.replace(0, np.nan).dropna()
    # Add closing trades one second after the close to be sure
    # they don't conflict with other trades executed at that time.
    end_dt = open_pos.name + pd.Timedelta(seconds=1)

    for sym, ending_val in open_pos.iteritems():
        txn_sym = transactions[transactions.symbol == sym]

        ending_amount = txn_sym.amount.sum()

        ending_price = ending_val / ending_amount
        closing_txn = {'symbol': sym,
                       'amount': -ending_amount,
                       'price': ending_price}

        closing_txn = pd.DataFrame(closing_txn, index=[end_dt])
        closed_txns = closed_txns.append(closing_txn)

    return closed_txns

def apply_sector_mappings_to_round_trips(round_trips, sector_mappings):
    """
    Translates round trip symbols to sectors.

    Parameters
    ----------
    round_trips : pd.DataFrame
        DataFrame with one row per round trip trade.
        - See full explanation in round_trips.extract_round_trips
    sector_mappings : dict or pd.Series, optional
        Security identifier to sector mapping.
        Security ids as keys, sectors as values.

    Returns
    -------
    sector_round_trips : pd.DataFrame
        Round trips with symbol names replaced by sector names.
    """

    sector_round_trips = round_trips.copy()
    sector_round_trips.symbol = sector_round_trips.symbol.apply(
        lambda x: sector_mappings.get(x, 'No Sector Mapping'))
    sector_round_trips = sector_round_trips.dropna(axis=0)

    return sector_round_trips
<commit_msg>BUG only keep relevant columns in transactions dataframe when appending closing transactions.<commit_after>#
# Copyright 2015 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import division
from collections import defaultdict

import pandas as pd
import numpy as np

def extract_round_trips(transactions):
    """
    Group transactions into "round trips." A round trip is started when a new
    long or short position is opened and is only completed when the number
    of shares in that position returns to or crosses zero.
    Computes pnl for each round trip.

    For example, the following transactions would constitute one round trip:
    index                  amount   price    symbol
    2004-01-09 12:18:01    186      324.12   'AAPL'
    2004-01-09 15:12:53    -10      344.54   'AAPL'
    2004-01-13 14:41:23    24       320.21   'AAPL'
    2004-01-30 10:23:34    -200     340.43   'AAPL'

    Parameters
    ----------
    transactions : pd.DataFrame
        Prices and amounts of executed trades. One row per trade.
        - See full explanation in tears.create_full_tear_sheet

    Returns
    -------
    round_trips : pd.DataFrame
        DataFrame with one row per round trip.
    """
    # Transactions that cross zero must be split into separate
    # long and short transactions that end/start on zero.
    transactions_split = split_trades(transactions)

    transactions_split['txn_dollars'] =  \
        -transactions_split['amount'] * transactions_split['price']

    round_trips = defaultdict(list)

    for sym, trans_sym in transactions_split.groupby('symbol'):
        trans_sym = trans_sym.sort_index()
        amount_cumsum = trans_sym.amount.cumsum()
        # Find indicies where the position amount returns to zero.
        closed_idx = np.where(amount_cumsum == 0)[0] + 1
        # Identify the first trade as the beginning of a round trip.
        closed_idx = np.insert(closed_idx, 0, 0)

        for trade_start, trade_end in zip(closed_idx, closed_idx[1:]):
            txn = trans_sym.iloc[trade_start:trade_end]

            if len(txn) == 0:
                continue

            assert txn.amount.sum() == 0
            long_trade = txn.amount.iloc[0] > 0
            pnl = txn.txn_dollars.sum()
            round_trips['symbol'].append(sym)
            round_trips['pnl'].append(pnl)
            round_trips['duration'].append(txn.index[-1] - txn.index[0])
            round_trips['long'].append(long_trade)
            round_trips['open_dt'].append(txn.index[0])
            round_trips['close_dt'].append(txn.index[-1])

            # Investing txns push the position amount farther from zero.
            # invested is always a positive value. Returned - Invested = PnL.
            if long_trade:
                invested = -txn.query('txn_dollars < 0').txn_dollars.sum()
            else:
                invested = txn.query('txn_dollars > 0').txn_dollars.sum()

            if invested == 0:
                round_trips['returns'].append(0)
            else:
                round_trips['returns'].append(pnl / invested)

    if len(round_trips) == 0:
        return pd.DataFrame([])

    round_trips = pd.DataFrame(round_trips)
    round_trips = round_trips[['open_dt', 'close_dt', 'duration',
                               'pnl', 'returns', 'long', 'symbol']]

    return round_trips

def split_trades(transactions):
    """
    Splits transactions that cause total position amount to cross zero.
    In other words, separates of the closing of one short/long position
    with the opening of a new long/short position.

    For example, the second transaction in this transactions DataFrame
    would be divided as shown in the second DataFrame:
    index                  amount   price    symbol
    2004-01-09 12:18:01    180      324.12   'AAPL'
    2004-01-09 15:12:53    -200     344.54   'AAPL'

    index                  amount   price    symbol
    2004-01-09 12:18:01    180      324.12   'AAPL'
    2004-01-09 15:12:53    -180     344.54   'AAPL'
    2004-01-09 15:12:54    -20      344.54   'AAPL'

    Parameters
    ----------
    transactions : pd.DataFrame
        Prices and amounts of executed trades. One row per trade.
        - See full explanation in tears.create_full_tear_sheet

    Returns
    -------
    transactions_split : pd.DataFrame
        Prices and amounts of executed trades. Trades that cause
        total position amount to cross zero are divided.
    """

    trans_split = []

    for sym, trans_sym in transactions.groupby('symbol'):
        trans_sym = trans_sym.sort_index()

        while True:
            cum_amount = trans_sym.amount.cumsum()
            # find the indicies where position amount crosses zero
            sign_flip = np.where(np.abs(np.diff(np.sign(cum_amount))) == 2)[0]

            if len(sign_flip) == 0:
                break  # all sign flips are converted

            sign_flip = sign_flip[0] + 2

            txn = trans_sym.iloc[:sign_flip]

            left_over_txn_amount = txn.amount.sum()
            assert left_over_txn_amount != 0

            split_txn_1 = txn.iloc[[-1]].copy()
            split_txn_2 = txn.iloc[[-1]].copy()

            split_txn_1['amount'] -= left_over_txn_amount
            split_txn_2['amount'] = left_over_txn_amount

            # Delay 2nd trade by a second to avoid overlapping indices
            split_txn_2.index += pd.Timedelta(seconds=1)

            assert split_txn_1.amount.iloc[0] + \
                split_txn_2.amount.iloc[0] == txn.iloc[-1].amount
            assert trans_sym.iloc[:sign_flip - 1].amount.sum() + \
                split_txn_1.amount.iloc[0] == 0

            # Recreate transactions so far with split transaction
            trans_sym = pd.concat([trans_sym.iloc[:sign_flip - 1],
                                   split_txn_1,
                                   split_txn_2,
                                   trans_sym.iloc[sign_flip:]])

        assert np.all(np.abs(np.diff(np.sign(trans_sym.amount.cumsum()))) != 2)
        trans_split.append(trans_sym)

    transactions_split = pd.concat(trans_split)

    return transactions_split

def add_closing_transactions(positions, transactions):
    """
    Appends transactions that close out all positions at the end of
    the timespan covered by positions data. Utilizes pricing information
    in the positions DataFrame to determine closing price.

    Parameters
    ----------
    positions : pd.DataFrame
        The positions that the strategy takes over time.
    transactions : pd.DataFrame
        Prices and amounts of executed trades. One row per trade.
        - See full explanation in tears.create_full_tear_sheet

    Returns
    -------
    closed_txns : pd.DataFrame
        Transactions with closing transactions appended.
    """

    closed_txns = transactions[['symbol', 'amount', 'price']]

    pos_at_end = positions.drop('cash', axis=1).iloc[-1]
    open_pos = pos_at_end.replace(0, np.nan).dropna()
    # Add closing trades one second after the close to be sure
    # they don't conflict with other trades executed at that time.
    end_dt = open_pos.name + pd.Timedelta(seconds=1)

    for sym, ending_val in open_pos.iteritems():
        txn_sym = transactions[transactions.symbol == sym]

        ending_amount = txn_sym.amount.sum()

        ending_price = ending_val / ending_amount
        closing_txn = {'symbol': sym,
                       'amount': -ending_amount,
                       'price': ending_price}

        closing_txn = pd.DataFrame(closing_txn, index=[end_dt])
        closed_txns = closed_txns.append(closing_txn)

    return closed_txns

def apply_sector_mappings_to_round_trips(round_trips, sector_mappings):
    """
    Translates round trip symbols to sectors.

    Parameters
    ----------
    round_trips : pd.DataFrame
        DataFrame with one row per round trip trade.
        - See full explanation in round_trips.extract_round_trips
    sector_mappings : dict or pd.Series, optional
        Security identifier to sector mapping.
        Security ids as keys, sectors as values.

    Returns
    -------
    sector_round_trips : pd.DataFrame
        Round trips with symbol names replaced by sector names.
    """

    sector_round_trips = round_trips.copy()
    sector_round_trips.symbol = sector_round_trips.symbol.apply(
        lambda x: sector_mappings.get(x, 'No Sector Mapping'))
    sector_round_trips = sector_round_trips.dropna(axis=0)

    return sector_round_trips
<|endoftext|>


<commit_before>import logging

from tg import expose, redirect, flash
from pylons import c, g
from pymongo.bson import ObjectId

from ming.orm import session

from pyforge.lib.helpers import push_config
from pyforge.lib.security import require, has_artifact_access
from pyforge import model

log = logging.getLogger(__name__)

class ConfigOption(object):

    def __init__(self, name, ming_type, default):
        self.name, self.ming_type, self._default = (
            name, ming_type, default)

    @property
    def default(self):
        if callable(self._default):
            return self._default()
        return self._default

class WSGIHook(object):

    def handles(self, environ):
        return False

    def __call__(self, environ, start_response):
        raise NotImplementedError, '__call__' # pragma no cover

class SitemapEntry(object):

    def __init__(self, label, url=None, children=None, className=None, ui_icon=None):
        self.label = label
        self.className = className
        self.url = url
        self.ui_icon = ui_icon
        if children is None:
            children = []
        self.children = children

    def __getitem__(self, x):
        if isinstance(x, (list, tuple)):
            self.children.extend(list(x))
        else:
            self.children.append(x)
        return self

    def bind_app(self, app):
        lbl = self.label
        url = self.url
        if callable(lbl):
            lbl = lbl(app)
        if url is not None and not url.startswith('/'):
            url = app.url + url
        return SitemapEntry(lbl, url, [
                ch.bind_app(app) for ch in self.children], className=self.className)

    def extend(self, sitemap):
        child_index = dict(
            (ch.label, ch) for ch in self.children)
        for e in sitemap:
            lbl = e.label
            match = child_index.get(e.label)
            if match and match.url == e.url:
                match.extend(e.children)
            else:
                self.children.append(e)
                child_index[lbl] = e

class WidgetController(object):
    widgets=[]

    def __init__(self, app): pass

    def portlet(self, content):
        return '
%s
' % content

class Application(object):
    'base pyforge pluggable application'
    __version__ = None
    config_options = [
        ConfigOption('mount_point', str, 'app') ]
    templates=None # path to templates
    script_name=None
    root=None  # root controller
    api_root=None
    permissions=[]
    sitemap = [ ]
    installable=True
    wsgi = WSGIHook()
    widget = WidgetController
    searchable = False
    DiscussionClass = model.Discussion
    PostClass = model.Post
    AttachmentClass = model.Attachment

    def __init__(self, project, app_config_object):
        self.project = project
        self.config = app_config_object # pragma: no cover
        self.admin = DefaultAdminController(self)
        self.url = self.config.url()

    def has_access(self, user, topic):
        '''Whether the user has access to send email to the given topic'''
        return False

    @classmethod
    def default_options(cls):
        return dict(
            (co.name, co.default)
            for co in cls.config_options)

    def install(self, project):
        'Whatever logic is required to initially set up a tool'
        # Create the discussion object
        discussion = self.DiscussionClass(
            shortname=self.config.options.mount_point,
            name='%s Discussion' % self.config.options.mount_point,
            description='Forum for %s comments' % self.config.options.mount_point)
        session(discussion).flush()
        self.config.discussion_id = discussion._id

    def uninstall(self, project=None, project_id=None):
        'Whatever logic is required to tear down a tool'
        if project_id is None: project_id = project._id
        # De-index all the artifacts belonging to this tool in one fell swoop
        g.solr.delete('project_id_s:%s AND mount_point_s:%s' % (
                project_id, self.config.options['mount_point']))
        # Remove all tags referring to this tool
        q_aref ={
            'artifact_ref.project_id':project_id,
            'artifact_ref.mount_point':self.config.options['mount_point']}
        model.Tag.query.remove(q_aref)
        model.TagEvent.query.remove(q_aref)
        model.UserTags.query.remove(q_aref)
        self.config.delete()

    def sidebar_menu(self):
        return []

    def admin_menu(self):
        admin_url = c.project.url()+'admin/'+self.config.options.mount_point+'/'
        links = []
        # if self.permissions and has_artifact_access('configure', app=self)():
        #     links.append(SitemapEntry('Permissions', admin_url + 'permissions', className='nav_child'))
        if len(self.config_options) > 1:
            links.append(SitemapEntry('Options', admin_url + 'options', className='nav_child'))
        return links

    def message_auditor(self, routing_key, data, artifact, **kw):
        # Find ancestor comment
        in_reply_to = data.get('in_reply_to', [])
        if in_reply_to:
            parent_id = in_reply_to[0]
        else:
            parent_id = None
        thd = artifact.discussion_thread(data)
        # Handle attachments
        message_id = data['message_id'][0]
        if data.get('filename'):
            log.info('Saving attachment %s', data['filename'])
            self.AttachmentClass.save(data['filename'],
                                  data.get('content_type', 'application/octet-stream'),
                                  data['payload'],
                                  discussion_id=self.config.discussion_id,
                                  post_id=message_id)
            return
        # Handle duplicates
        original = self.PostClass.query.get(_id=message_id)
        if original:
            log.info('Saving text attachment')
            self.AttachmentClass.save('alternate',
                                  data.get('content_type', 'application/octet-stream'),
                                  data['payload'],
                                  discussion_id=self.config.discussion_id,
                                  post_id=message_id)
            return
        thd.post(
            text=data['payload'],
            message_id=message_id,
            parent_id=parent_id,
            **kw)

class DefaultAdminController(object):

    def __init__(self, app):
        self.app = app

    def index(self):
        return redirect('permissions')

    @expose('pyforge.templates.app_admin_permissions')
    def permissions(self):
        return dict(app=self.app,
                    allow_config=has_artifact_access('configure', app=self.app)())

    @expose('pyforge.templates.app_admin_options')
    def options(self):
        return dict(app=self.app,
                    allow_config=has_artifact_access('configure', app=self.app)())

    @expose()
    def configure(self, **kw):
        with push_config(c, app=self.app):
            require(has_artifact_access('configure', app=self.app), 'Must have configure permission')
            is_admin = self.app.config.tool_name == 'admin'
            if kw.pop('delete', False):
                if is_admin:
                    flash('Cannot delete the admin tool, sorry....')
                    redirect('.')
                c.project.uninstall_app(self.app.config.options.mount_point)
                redirect('..')
            for k,v in kw.iteritems():
                self.app.config.options[k] = v
            if is_admin:
                # possibly moving admin mount point
                redirect('/'
                         + c.project._id
                         + self.app.config.options.mount_point
                         + '/'
                         + self.app.config.options.mount_point
                         + '/')
            else:
                redirect('../' + self.app.config.options.mount_point + '/')

    @expose()
    def add_perm(self, permission, role):
        require(has_artifact_access('configure', app=self.app))
        self.app.config.acl[permission].append(ObjectId(role))
        redirect('permissions')

    @expose()
    def del_perm(self, permission, role):
        require(has_artifact_access('configure', app=self.app))
        self.app.config.acl[permission].remove(ObjectId(role))
        redirect('permissions')

<commit_msg>[#371] - Fix error that places None in posts when posting via email with no text body<commit_after>import logging

from tg import expose, redirect, flash
from pylons import c, g
from pymongo.bson import ObjectId

from ming.orm import session

from pyforge.lib.helpers import push_config
from pyforge.lib.security import require, has_artifact_access
from pyforge import model

log = logging.getLogger(__name__)

class ConfigOption(object):

    def __init__(self, name, ming_type, default):
        self.name, self.ming_type, self._default = (
            name, ming_type, default)

    @property
    def default(self):
        if callable(self._default):
            return self._default()
        return self._default

class WSGIHook(object):

    def handles(self, environ):
        return False

    def __call__(self, environ, start_response):
        raise NotImplementedError, '__call__' # pragma no cover

class SitemapEntry(object):

    def __init__(self, label, url=None, children=None, className=None, ui_icon=None):
        self.label = label
        self.className = className
        self.url = url
        self.ui_icon = ui_icon
        if children is None:
            children = []
        self.children = children

    def __getitem__(self, x):
        if isinstance(x, (list, tuple)):
            self.children.extend(list(x))
        else:
            self.children.append(x)
        return self

    def bind_app(self, app):
        lbl = self.label
        url = self.url
        if callable(lbl):
            lbl = lbl(app)
        if url is not None and not url.startswith('/'):
            url = app.url + url
        return SitemapEntry(lbl, url, [
                ch.bind_app(app) for ch in self.children], className=self.className)

    def extend(self, sitemap):
        child_index = dict(
            (ch.label, ch) for ch in self.children)
        for e in sitemap:
            lbl = e.label
            match = child_index.get(e.label)
            if match and match.url == e.url:
                match.extend(e.children)
            else:
                self.children.append(e)
                child_index[lbl] = e

class WidgetController(object):
    widgets=[]

    def __init__(self, app): pass

    def portlet(self, content):
        return '
%s
' % content

class Application(object):
    'base pyforge pluggable application'
    __version__ = None
    config_options = [
        ConfigOption('mount_point', str, 'app') ]
    templates=None # path to templates
    script_name=None
    root=None  # root controller
    api_root=None
    permissions=[]
    sitemap = [ ]
    installable=True
    wsgi = WSGIHook()
    widget = WidgetController
    searchable = False
    DiscussionClass = model.Discussion
    PostClass = model.Post
    AttachmentClass = model.Attachment

    def __init__(self, project, app_config_object):
        self.project = project
        self.config = app_config_object # pragma: no cover
        self.admin = DefaultAdminController(self)
        self.url = self.config.url()

    def has_access(self, user, topic):
        '''Whether the user has access to send email to the given topic'''
        return False

    @classmethod
    def default_options(cls):
        return dict(
            (co.name, co.default)
            for co in cls.config_options)

    def install(self, project):
        'Whatever logic is required to initially set up a tool'
        # Create the discussion object
        discussion = self.DiscussionClass(
            shortname=self.config.options.mount_point,
            name='%s Discussion' % self.config.options.mount_point,
            description='Forum for %s comments' % self.config.options.mount_point)
        session(discussion).flush()
        self.config.discussion_id = discussion._id

    def uninstall(self, project=None, project_id=None):
        'Whatever logic is required to tear down a tool'
        if project_id is None: project_id = project._id
        # De-index all the artifacts belonging to this tool in one fell swoop
        g.solr.delete('project_id_s:%s AND mount_point_s:%s' % (
                project_id, self.config.options['mount_point']))
        # Remove all tags referring to this tool
        q_aref ={
            'artifact_ref.project_id':project_id,
            'artifact_ref.mount_point':self.config.options['mount_point']}
        model.Tag.query.remove(q_aref)
        model.TagEvent.query.remove(q_aref)
        model.UserTags.query.remove(q_aref)
        self.config.delete()

    def sidebar_menu(self):
        return []

    def admin_menu(self):
        admin_url = c.project.url()+'admin/'+self.config.options.mount_point+'/'
        links = []
        # if self.permissions and has_artifact_access('configure', app=self)():
        #     links.append(SitemapEntry('Permissions', admin_url + 'permissions', className='nav_child'))
        if len(self.config_options) > 1:
            links.append(SitemapEntry('Options', admin_url + 'options', className='nav_child'))
        return links

    def message_auditor(self, routing_key, data, artifact, **kw):
        # Find ancestor comment
        in_reply_to = data.get('in_reply_to', [])
        if in_reply_to:
            parent_id = in_reply_to[0]
        else:
            parent_id = None
        thd = artifact.discussion_thread(data)
        # Handle attachments
        message_id = data['message_id'][0]
        if data.get('filename'):
            log.info('Saving attachment %s', data['filename'])
            self.AttachmentClass.save(data['filename'],
                                  data.get('content_type', 'application/octet-stream'),
                                  data['payload'],
                                  discussion_id=self.config.discussion_id,
                                  post_id=message_id)
            return
        # Handle duplicates
        original = self.PostClass.query.get(_id=message_id)
        if original:
            log.info('Saving text attachment')
            self.AttachmentClass.save('alternate',
                                  data.get('content_type', 'application/octet-stream'),
                                  data['payload'],
                                  discussion_id=self.config.discussion_id,
                                  post_id=message_id)
            return
        thd.post(
            text=data['payload'] or '--no text body--',
            message_id=message_id,
            parent_id=parent_id,
            **kw)

class DefaultAdminController(object):

    def __init__(self, app):
        self.app = app

    def index(self):
        return redirect('permissions')

    @expose('pyforge.templates.app_admin_permissions')
    def permissions(self):
        return dict(app=self.app,
                    allow_config=has_artifact_access('configure', app=self.app)())

    @expose('pyforge.templates.app_admin_options')
    def options(self):
        return dict(app=self.app,
                    allow_config=has_artifact_access('configure', app=self.app)())

    @expose()
    def configure(self, **kw):
        with push_config(c, app=self.app):
            require(has_artifact_access('configure', app=self.app), 'Must have configure permission')
            is_admin = self.app.config.tool_name == 'admin'
            if kw.pop('delete', False):
                if is_admin:
                    flash('Cannot delete the admin tool, sorry....')
                    redirect('.')
                c.project.uninstall_app(self.app.config.options.mount_point)
                redirect('..')
            for k,v in kw.iteritems():
                self.app.config.options[k] = v
            if is_admin:
                # possibly moving admin mount point
                redirect('/'
                         + c.project._id
                         + self.app.config.options.mount_point
                         + '/'
                         + self.app.config.options.mount_point
                         + '/')
            else:
                redirect('../' + self.app.config.options.mount_point + '/')

    @expose()
    def add_perm(self, permission, role):
        require(has_artifact_access('configure', app=self.app))
        self.app.config.acl[permission].append(ObjectId(role))
        redirect('permissions')

    @expose()
    def del_perm(self, permission, role):
        require(has_artifact_access('configure', app=self.app))
        self.app.config.acl[permission].remove(ObjectId(role))
        redirect('permissions')

<|endoftext|>


<commit_before>import re

from livestreamer.compat import urlparse
from livestreamer.plugin import Plugin
from livestreamer.plugin.api import http, validate
from livestreamer.stream import HLSStream, HTTPStream, RTMPStream

LIVE_API = "http://www.hitbox.tv/api/media/live/{0}?showHidden=true"
PLAYER_API = "http://www.hitbox.tv/api/player/config/{0}/{1}?embed=false&showHidden=true"
SWF_BASE = "http://edge.vie.hitbox.tv/static/player/flowplayer/"
SWF_URL = SWF_BASE + "flowplayer.commercial-3.2.16.swf"
VOD_BASE_URL = "http://edge.bf.hitbox.tv/static/videos/vods"

_quality_re = re.compile("(\d+p)$")
_url_re = re.compile("""
    http(s)?://(www\.)?hitbox.tv
    /(?P<channel>[^/]+)
    (?:
        /(?P<media_id>[^/]+)
    )?
""", re.VERBOSE)

_live_schema = validate.Schema(
    {
        "livestream": [{
            "media_is_live": validate.all(
                validate.text,
                validate.transform(int),
                validate.transform(bool)
            ),
            "media_id": validate.text
        }],
    },
    validate.get("livestream"),
    validate.length(1),
    validate.get(0)
)
_player_schema = validate.Schema(
    {
        "clip": {
            "baseUrl": validate.any(None, validate.text),
            "bitrates": validate.all(
                validate.filter(lambda b: b.get("url") and b.get("label")),
                [{
                    "label": validate.text,
                    "url": validate.text
                }],
            )
        },
        validate.optional("playlist"): [{
            validate.optional("connectionProvider"): validate.text,
            validate.optional("netConnectionUrl"): validate.text,
            validate.optional("bitrates"): [{
                "label": validate.text,
                "url": validate.text
            }]
        }],
        "plugins": {
            validate.optional("clustering"): {
                "netConnectionUrl": validate.text,
                "url": validate.text
            }
        }
    }
)

class Hitbox(Plugin):
    @classmethod
    def can_handle_url(self, url):
        return _url_re.match(url)

    def _get_quality(self, label):
        match = _quality_re.search(label)
        if match:
            return match.group(1)

        return "live"

    def _get_streams(self):
        match = _url_re.match(self.url)
        if not match:
            return

        channel, media_id = match.group("channel", "media_id")
        if channel != "video":
            res = http.get(LIVE_API.format(channel))
            livestream = http.json(res, schema=_live_schema)
            if not livestream["media_is_live"]:
                return

            media_id = livestream["media_id"]
            media_type = "live"
        else:
            media_type = "video"

        res = http.get(PLAYER_API.format(media_type, media_id))
        player = http.json(res, schema=_player_schema)
        if media_type == "live":
            swf_url = SWF_URL
            for playlist in player.get("playlist", []):
                bitrates = playlist.get("bitrates")
                provider = playlist.get("connectionProvider")
                rtmp = None

                if bitrates:
                    rtmp = playlist.get("netConnectionUrl")
                elif provider and provider in player["plugins"]:
                    provider = player["plugins"][provider]
                    swf_name = provider["url"]
                    swf_url = SWF_BASE + swf_name
                    rtmp = provider["netConnectionUrl"]
                    bitrates = player["clip"]["bitrates"]
                else:
                    continue

                for bitrate in bitrates:
                    quality = self._get_quality(bitrate["label"])
                    url = bitrate["url"]
                    stream = RTMPStream(self.session, {
                        "rtmp": rtmp,
                        "pageUrl": self.url,
                        "playpath": url,
                        "swfVfy": swf_url,
                        "live": True
                    })
                    yield quality, stream
        else:
            base_url = player["clip"]["baseUrl"] or VOD_BASE_URL
            for bitrate in player["clip"]["bitrates"]:
                url = base_url + "/" + bitrate["url"]
                quality = self._get_quality(bitrate["label"])

                if urlparse(url).path.endswith("m3u8"):
                    stream = HLSStream(self.session, url)
                else:
                    stream = HTTPStream(self.session, url)

                yield quality, stream

__plugin__ = Hitbox
<commit_msg>plugins.hitbox: Refactor to use the StreamMapper.<commit_after>import re

from itertools import chain

from livestreamer.compat import urlparse
from livestreamer.plugin import Plugin
from livestreamer.plugin.api import StreamMapper, http, validate
from livestreamer.stream import HLSStream, HTTPStream, RTMPStream

HLS_PLAYLIST_BASE = "http://www.hitbox.tv{0}"
LIVE_API = "http://www.hitbox.tv/api/media/live/{0}?showHidden=true"
PLAYER_API = "http://www.hitbox.tv/api/player/config/{0}/{1}?embed=false&showHidden=true"
SWF_BASE = "http://edge.vie.hitbox.tv/static/player/flowplayer/"
SWF_URL = SWF_BASE + "flowplayer.commercial-3.2.16.swf"
VOD_BASE_URL = "http://edge.bf.hitbox.tv/static/videos/vods"

_quality_re = re.compile("(\d+p)$")
_url_re = re.compile("""
    http(s)?://(www\.)?hitbox.tv
    /(?P<channel>[^/]+)
    (?:
        /(?P<media_id>[^/]+)
    )?
""", re.VERBOSE)

_live_schema = validate.Schema(
    {
        "livestream": [{
            "media_is_live": validate.all(
                validate.text,
                validate.transform(int),
                validate.transform(bool)
            ),
            "media_id": validate.text
        }],
    },
    validate.get("livestream"),
    validate.length(1),
    validate.get(0)
)
_player_schema = validate.Schema(
    {
        "clip": {
            "baseUrl": validate.any(None, validate.text),
            "bitrates": validate.all(
                validate.filter(lambda b: b.get("url") and b.get("label")),
                [{
                    "label": validate.text,
                    "url": validate.text,
                }],
            )
        },
        validate.optional("playlist"): [{
            validate.optional("connectionProvider"): validate.text,
            validate.optional("netConnectionUrl"): validate.text,
            validate.optional("bitrates"): [{
                "label": validate.text,
                "url": validate.text,
                "provider": validate.text
            }]
        }],
        "plugins": validate.all(
            dict,
            validate.filter(lambda k, v: k in ["rtmp", "rtmpHitbox", "hls"]),
            {
                validate.text: {
                    validate.optional("netConnectionUrl"): validate.text,
                    "url": validate.text
                }
            }
        )
    }
)

class Hitbox(Plugin):
    @classmethod
    def can_handle_url(self, url):
        return _url_re.match(url)

    def _get_quality(self, label):
        match = _quality_re.search(label)
        if match:
            return match.group(1)

        return "live"

    def _create_hls_streams(self, bitrate):
        url = bitrate["url"]
        quality = self._get_quality(bitrate["label"])

        if not url.startswith("http"):
            url = HLS_PLAYLIST_BASE.format(url)

        if bitrate["label"] == "Auto":
            try:
                streams = HLSStream.parse_variant_playlist(self.session, url)
                return streams.items()
            except IOError as err:
                self.logger.warning("Failed to extract HLS streams: {0}", err)
        else:
            return quality, HLSStream(self.session, url)

    def _create_rtmp_stream(self, rtmp, swf_url, bitrate):
        quality = self._get_quality(bitrate["label"])
        url = bitrate["url"]
        stream = RTMPStream(self.session, {
            "rtmp": rtmp,
            "pageUrl": self.url,
            "playpath": url,
            "swfVfy": swf_url,
            "live": True
        })

        return quality, stream

    def _get_live_streams(self, player):
        mappers = []
        swf_url = SWF_URL
        for playlist in player.get("playlist", []):
            bitrates = playlist.get("bitrates")
            provider = playlist.get("connectionProvider")
            rtmp = None

            if bitrates:
                rtmp = playlist.get("netConnectionUrl")
            elif provider and provider in player["plugins"]:
                provider = player["plugins"][provider]
                swf_name = provider["url"]
                swf_url = SWF_BASE + swf_name
                rtmp = provider["netConnectionUrl"]
                bitrates = player["clip"]["bitrates"]
            else:
                continue

            mapper = StreamMapper(
                cmp=lambda provider, bitrate: bitrate["provider"].startswith(provider)
            )
            mapper.map("hls", self._create_hls_streams)
            mapper.map("rtmp", self._create_rtmp_stream, rtmp, swf_url)
            mappers.append(mapper(bitrates))

        return chain.from_iterable(mappers)

    def _create_video_stream(self, cls, base_url, bitrate):
        url = base_url + "/" + bitrate["url"]
        quality = self._get_quality(bitrate["label"])
        return quality, cls(self.session, url)

    def _get_video_streams(self, player):
        base_url = player["clip"]["baseUrl"] or VOD_BASE_URL
        mapper = StreamMapper(
            cmp=lambda ext, bitrate: urlparse(bitrate["url"]).path.endswith(ext)
        )
        mapper.map(".m3u8", self._create_video_stream, HLSStream, base_url)
        mapper.map(".mp4", self._create_video_stream, HTTPStream, base_url)
        mapper.map(".flv", self._create_video_stream, HTTPStream, base_url)

        return mapper(player["clip"]["bitrates"])

    def _get_streams(self):
        match = _url_re.match(self.url)
        if not match:
            return

        channel, media_id = match.group("channel", "media_id")
        if channel != "video":
            res = http.get(LIVE_API.format(channel))
            livestream = http.json(res, schema=_live_schema)
            if not livestream["media_is_live"]:
                return

            media_id = livestream["media_id"]
            media_type = "live"
        else:
            media_type = "video"

        res = http.get(PLAYER_API.format(media_type, media_id))
        player = http.json(res, schema=_player_schema)

        if media_type == "live":
            return self._get_live_streams(player)
        else:
            return self._get_video_streams(player)

__plugin__ = Hitbox
<|endoftext|>


<commit_before>from os.path import abspath, dirname
import argparse
import inspect
import logging
from pkg_resources import iter_entry_points

try:
    from importlib.metadata import metadata
except ImportError:
    from importlib_metadata import metadata

logger = logging.getLogger("pyfrc.sim")

class PyFrcSim:
    """
    Runs the robot using WPILib's GUI HAL Simulator
    """

    def __init__(self, parser: argparse.ArgumentParser):
        parser.add_argument(
            "--nogui",
            default=False,
            action="store_true",
            help="Don't use the WPIlib simulation gui",
        )

        self.simexts = {}

        for entry_point in iter_entry_points(group="robotpysimext", name=None):
            if entry_point.module_name == "halsim_gui":
                continue
            try:
                sim_ext_module = entry_point.load()
            except ImportError:
                print(f"WARNING: Error detected in {entry_point}")
                continue

            self.simexts[entry_point.name] = sim_ext_module

            parser.add_argument(
                f"--{entry_point.name}",
                default=False,
                action="store_true",
                help=metadata(entry_point.dist.project_name)["summary"],
            )

    def run(self, options, robot_class, **static_options):

        if not options.nogui:
            try:
                import halsim_gui
            except ImportError:
                print("robotpy-halsim-gui is not installed!")
                exit(1)
            else:
                halsim_gui.loadExtension()

        for name, module in self.simexts.items():
            if getattr(options, name.replace("-", "_"), False):
                try:
                    module.loadExtension()
                except:
                    print(f"Error loading {name}!")
                    raise

        # initialize physics, attach to the user robot class
        from ..physics.core import PhysicsInterface, PhysicsInitException

        robot_file = abspath(inspect.getfile(robot_class))
        robot_path = dirname(robot_file)

        try:
            physics = PhysicsInterface._create(robot_path)
            if physics:
                robot_class._simulationInit = physics._simulationInit
                robot_class._simulationPeriodic = physics._simulationPeriodic

            # run the robot
            return robot_class.main(robot_class)

        except PhysicsInitException:
            return False
<commit_msg>Save current working directory when loading extensions<commit_after>import os
from os.path import abspath, dirname
import argparse
import inspect
import logging
from pkg_resources import iter_entry_points

try:
    from importlib.metadata import metadata
except ImportError:
    from importlib_metadata import metadata

logger = logging.getLogger("pyfrc.sim")

class PyFrcSim:
    """
    Runs the robot using WPILib's GUI HAL Simulator
    """

    def __init__(self, parser: argparse.ArgumentParser):
        parser.add_argument(
            "--nogui",
            default=False,
            action="store_true",
            help="Don't use the WPIlib simulation gui",
        )

        self.simexts = {}

        for entry_point in iter_entry_points(group="robotpysimext", name=None):
            if entry_point.module_name == "halsim_gui":
                continue
            try:
                sim_ext_module = entry_point.load()
            except ImportError:
                print(f"WARNING: Error detected in {entry_point}")
                continue

            self.simexts[entry_point.name] = sim_ext_module

            parser.add_argument(
                f"--{entry_point.name}",
                default=False,
                action="store_true",
                help=metadata(entry_point.dist.project_name)["summary"],
            )

    def run(self, options, robot_class, **static_options):

        if not options.nogui:
            try:
                import halsim_gui
            except ImportError:
                print("robotpy-halsim-gui is not installed!")
                exit(1)
            else:
                halsim_gui.loadExtension()

        # Some extensions (gui) changes the current directory
        cwd = os.getcwd()

        for name, module in self.simexts.items():
            if getattr(options, name.replace("-", "_"), False):
                try:
                    module.loadExtension()
                except:
                    print(f"Error loading {name}!")
                    raise

        os.chdir(cwd)

        # initialize physics, attach to the user robot class
        from ..physics.core import PhysicsInterface, PhysicsInitException

        robot_file = abspath(inspect.getfile(robot_class))
        robot_path = dirname(robot_file)

        try:
            physics = PhysicsInterface._create(robot_path)
            if physics:
                robot_class._simulationInit = physics._simulationInit
                robot_class._simulationPeriodic = physics._simulationPeriodic

            # run the robot
            return robot_class.main(robot_class)

        except PhysicsInitException:
            return False
<|endoftext|>