<commit_before>#
# Copyright (C) 2003 Stefan Seefeld
# All rights reserved.
# Licensed to the public under the terms of the GNU LGPL (>= 2),
# see the file COPYING for details.
#

from Processor import Processor
import AST
from getoptions import get_options

import sys

def error(msg):
   """Write an error message and exit."""
   sys.stderr.write(msg)
   sys.stderr.write('\n')
   sys.exit(-1)

def process(argv = sys.argv, **commands):
   """Accept a set of commands and process according to command line options.
   The typical call will start with the name of the processor to be executed,
   followed by a set of parameters, followed by non-parameter arguments.
   All parameters are either of the form 'name=value', or '--name=value'.
   The first form expects 'value' to be valid python, the second a string.
   The remaining non-parameter arguments are associated with the 'input'
   parameter.
   Once this initialization is done, the named command's 'process' method
   is executed.
   """

   #first make sure the function was called with the correct argument types
   for c in commands:
      if not isinstance(commands[c], Processor):
         error("command '%s' isn't a valid processor"%c)

   if len(argv) < 2:
      error("Usage : %s <command> [args] [input files]"%argv[0])

   elif argv[1] == '--help':
      print "Usage: %s --help"%argv[0]
      print "   or: %s <command> --help"%argv[0]
      print "   or: %s <command> [parameters]"%argv[0]
      print ""
      print "Available commands:"
      for c in commands:
         print "   %s"%c
      sys.exit(0)

   command = argv[1]
   args = argv[2:]

   if '--help' in args:
      print "Parameters for command '%s'"%command
      parameters = commands[command].get_parameters()
      tab = max(map(lambda x:len(x), parameters.keys()))
      for p in parameters:
         print "   %-*s     %s"%(tab, p, parameters[p].doc)
      sys.exit(0)

   props = {}
   # process all option arguments...
   for o, a in get_options(args): props[o] = a

   # ...and keep remaining (non-option) arguments as 'input'
   if args: props['input'] = args

   if command in commands:
      ast = AST.AST()
      try:
         commands[command].process(ast, **props)
      except KeyError, e:
         error('missing argument "%s"'%e)
      except Processor.Error, e:
         error(str(e))
      except KeyboardInterrupt, e:
         print 'KeyboardInterrupt'
   else:
      error('no command "%s"'%command)

<commit_msg>Fix typo.<commit_after>#
# Copyright (C) 2003 Stefan Seefeld
# All rights reserved.
# Licensed to the public under the terms of the GNU LGPL (>= 2),
# see the file COPYING for details.
#

from Processor import Processor, Error
import AST
from getoptions import get_options

import sys

def error(msg):
   """Write an error message and exit."""
   sys.stderr.write(msg)
   sys.stderr.write('\n')
   sys.exit(-1)

def process(argv = sys.argv, **commands):
   """Accept a set of commands and process according to command line options.
   The typical call will start with the name of the processor to be executed,
   followed by a set of parameters, followed by non-parameter arguments.
   All parameters are either of the form 'name=value', or '--name=value'.
   The first form expects 'value' to be valid python, the second a string.
   The remaining non-parameter arguments are associated with the 'input'
   parameter.
   Once this initialization is done, the named command's 'process' method
   is executed.
   """

   #first make sure the function was called with the correct argument types
   for c in commands:
      if not isinstance(commands[c], Processor):
         error("command '%s' isn't a valid processor"%c)

   if len(argv) < 2:
      error("Usage : %s <command> [args] [input files]"%argv[0])

   elif argv[1] == '--help':
      print "Usage: %s --help"%argv[0]
      print "   or: %s <command> --help"%argv[0]
      print "   or: %s <command> [parameters]"%argv[0]
      print ""
      print "Available commands:"
      for c in commands:
         print "   %s"%c
      sys.exit(0)

   command = argv[1]
   args = argv[2:]

   if '--help' in args:
      print "Parameters for command '%s'"%command
      parameters = commands[command].get_parameters()
      tab = max(map(lambda x:len(x), parameters.keys()))
      for p in parameters:
         print "   %-*s     %s"%(tab, p, parameters[p].doc)
      sys.exit(0)

   props = {}
   # process all option arguments...
   for o, a in get_options(args): props[o] = a

   # ...and keep remaining (non-option) arguments as 'input'
   if args: props['input'] = args

   if command in commands:
      ast = AST.AST()
      try:
         commands[command].process(ast, **props)
      except KeyError, e:
         error('missing argument "%s"'%e)
      except Error, e:
         error(str(e))
      except KeyboardInterrupt, e:
         print 'KeyboardInterrupt'
   else:
      error('no command "%s"'%command)

<|endoftext|>


<commit_before># Copyright (c) 2013 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from neutron.agent import securitygroups_rpc as sg_rpc
from neutron.common import constants as q_const
from neutron.common import rpc as q_rpc
from neutron.common import topics
from neutron.db import agents_db
from neutron.db import api as db_api
from neutron.db import dhcp_rpc_base
from neutron.db import l3_rpc_base
from neutron.db import securitygroups_rpc_base as sg_db_rpc
from neutron.openstack.common import log
from neutron.openstack.common.rpc import proxy
from neutron.plugins.ml2 import db
from neutron.plugins.ml2 import driver_api as api
from neutron.plugins.ml2.drivers import type_tunnel
# REVISIT(kmestery): Allow the type and mechanism drivers to supply the
# mixins and eventually remove the direct dependencies on type_tunnel.

LOG = log.getLogger(__name__)

TAP_DEVICE_PREFIX = 'tap'
TAP_DEVICE_PREFIX_LENGTH = 3

class RpcCallbacks(dhcp_rpc_base.DhcpRpcCallbackMixin,
                   l3_rpc_base.L3RpcCallbackMixin,
                   sg_db_rpc.SecurityGroupServerRpcCallbackMixin,
                   type_tunnel.TunnelRpcCallbackMixin):

    RPC_API_VERSION = '1.1'
    # history
    #   1.0 Initial version (from openvswitch/linuxbridge)
    #   1.1 Support Security Group RPC

    def __init__(self, notifier, type_manager):
        # REVISIT(kmestery): This depends on the first three super classes
        # not having their own __init__ functions. If an __init__() is added
        # to one, this could break. Fix this and add a unit test to cover this
        # test in H3.
        super(RpcCallbacks, self).__init__(notifier, type_manager)

    def create_rpc_dispatcher(self):
        '''Get the rpc dispatcher for this manager.

        If a manager would like to set an rpc API version, or support more than
        one class as the target of rpc messages, override this method.
        '''
        return q_rpc.PluginRpcDispatcher([self,
                                          agents_db.AgentExtRpcCallback()])

    @classmethod
    def _device_to_port_id(cls, device):
        # REVISIT(rkukura): Consider calling into MechanismDrivers to
        # process device names, or having MechanismDrivers supply list
        # of device prefixes to strip.
        if device.startswith(TAP_DEVICE_PREFIX):
            return device[TAP_DEVICE_PREFIX_LENGTH:]
        else:
            return device

    @classmethod
    def get_port_from_device(cls, device):
        port_id = cls._device_to_port_id(device)
        port = db.get_port_and_sgs(port_id)
        if port:
            port['device'] = device
        return port

    def get_device_details(self, rpc_context, **kwargs):
        """Agent requests device details."""
        agent_id = kwargs.get('agent_id')
        device = kwargs.get('device')
        LOG.debug(_("Device %(device)s details requested by agent "
                    "%(agent_id)s"),
                  {'device': device, 'agent_id': agent_id})
        port_id = self._device_to_port_id(device)

        session = db_api.get_session()
        with session.begin(subtransactions=True):
            port = db.get_port(session, port_id)
            if not port:
                LOG.warning(_("Device %(device)s requested by agent "
                              "%(agent_id)s not found in database"),
                            {'device': device, 'agent_id': agent_id})
                return {'device': device}
            segments = db.get_network_segments(session, port.network_id)
            if not segments:
                LOG.warning(_("Device %(device)s requested by agent "
                              "%(agent_id)s has network %(network_id) with "
                              "no segments"),
                            {'device': device,
                             'agent_id': agent_id,
                             'network_id': port.network_id})
                return {'device': device}
            #TODO(rkukura): Use/create port binding
            segment = segments[0]
            new_status = (q_const.PORT_STATUS_ACTIVE if port.admin_state_up
                          else q_const.PORT_STATUS_DOWN)
            if port.status != new_status:
                port.status = new_status
            entry = {'device': device,
                     'network_id': port.network_id,
                     'port_id': port.id,
                     'admin_state_up': port.admin_state_up,
                     'network_type': segment[api.NETWORK_TYPE],
                     'segmentation_id': segment[api.SEGMENTATION_ID],
                     'physical_network': segment[api.PHYSICAL_NETWORK]}
            LOG.debug(_("Returning: %s"), entry)
            return entry

    def update_device_down(self, rpc_context, **kwargs):
        """Device no longer exists on agent."""
        # TODO(garyk) - live migration and port status
        agent_id = kwargs.get('agent_id')
        device = kwargs.get('device')
        LOG.debug(_("Device %(device)s no longer exists at agent "
                    "%(agent_id)s"),
                  {'device': device, 'agent_id': agent_id})
        port_id = self._device_to_port_id(device)

        session = db_api.get_session()
        with session.begin(subtransactions=True):
            port = db.get_port(session, port_id)
            if not port:
                LOG.warning(_("Device %(device)s updated down by agent "
                              "%(agent_id)s not found in database"),
                            {'device': device, 'agent_id': agent_id})
                return {'device': device,
                        'exists': False}
            if port.status != q_const.PORT_STATUS_DOWN:
                port.status = q_const.PORT_STATUS_DOWN
            return {'device': device,
                    'exists': True}

    def update_device_up(self, rpc_context, **kwargs):
        """Device is up on agent."""
        agent_id = kwargs.get('agent_id')
        device = kwargs.get('device')
        LOG.debug(_("Device %(device)s up at agent %(agent_id)s"),
                  {'device': device, 'agent_id': agent_id})
        port_id = self._device_to_port_id(device)

        session = db_api.get_session()
        with session.begin(subtransactions=True):
            port = db.get_port(session, port_id)
            if not port:
                LOG.warning(_("Device %(device)s updated up by agent "
                              "%(agent_id)s not found in database"),
                            {'device': device, 'agent_id': agent_id})
            if port.status != q_const.PORT_STATUS_ACTIVE:
                port.status = q_const.PORT_STATUS_ACTIVE

class AgentNotifierApi(proxy.RpcProxy,
                       sg_rpc.SecurityGroupAgentRpcApiMixin,
                       type_tunnel.TunnelAgentRpcApiMixin):
    """Agent side of the openvswitch rpc API.

    API version history:
        1.0 - Initial version.
        1.1 - Added get_active_networks_info, create_dhcp_port,
              update_dhcp_port, and removed get_dhcp_port methods.

    """

    BASE_RPC_API_VERSION = '1.1'

    def __init__(self, topic):
        super(AgentNotifierApi, self).__init__(
            topic=topic, default_version=self.BASE_RPC_API_VERSION)
        self.topic_network_delete = topics.get_topic_name(topic,
                                                          topics.NETWORK,
                                                          topics.DELETE)
        self.topic_port_update = topics.get_topic_name(topic,
                                                       topics.PORT,
                                                       topics.UPDATE)

    def network_delete(self, context, network_id):
        self.fanout_cast(context,
                         self.make_msg('network_delete',
                                       network_id=network_id),
                         topic=self.topic_network_delete)

    def port_update(self, context, port, network_type, segmentation_id,
                    physical_network):
        self.fanout_cast(context,
                         self.make_msg('port_update',
                                       port=port,
                                       network_type=network_type,
                                       segmentation_id=segmentation_id,
                                       physical_network=physical_network),
                         topic=self.topic_port_update)
<commit_msg>Corrects error in ml2 rpc log message formatting<commit_after># Copyright (c) 2013 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from neutron.agent import securitygroups_rpc as sg_rpc
from neutron.common import constants as q_const
from neutron.common import rpc as q_rpc
from neutron.common import topics
from neutron.db import agents_db
from neutron.db import api as db_api
from neutron.db import dhcp_rpc_base
from neutron.db import l3_rpc_base
from neutron.db import securitygroups_rpc_base as sg_db_rpc
from neutron.openstack.common import log
from neutron.openstack.common.rpc import proxy
from neutron.plugins.ml2 import db
from neutron.plugins.ml2 import driver_api as api
from neutron.plugins.ml2.drivers import type_tunnel
# REVISIT(kmestery): Allow the type and mechanism drivers to supply the
# mixins and eventually remove the direct dependencies on type_tunnel.

LOG = log.getLogger(__name__)

TAP_DEVICE_PREFIX = 'tap'
TAP_DEVICE_PREFIX_LENGTH = 3

class RpcCallbacks(dhcp_rpc_base.DhcpRpcCallbackMixin,
                   l3_rpc_base.L3RpcCallbackMixin,
                   sg_db_rpc.SecurityGroupServerRpcCallbackMixin,
                   type_tunnel.TunnelRpcCallbackMixin):

    RPC_API_VERSION = '1.1'
    # history
    #   1.0 Initial version (from openvswitch/linuxbridge)
    #   1.1 Support Security Group RPC

    def __init__(self, notifier, type_manager):
        # REVISIT(kmestery): This depends on the first three super classes
        # not having their own __init__ functions. If an __init__() is added
        # to one, this could break. Fix this and add a unit test to cover this
        # test in H3.
        super(RpcCallbacks, self).__init__(notifier, type_manager)

    def create_rpc_dispatcher(self):
        '''Get the rpc dispatcher for this manager.

        If a manager would like to set an rpc API version, or support more than
        one class as the target of rpc messages, override this method.
        '''
        return q_rpc.PluginRpcDispatcher([self,
                                          agents_db.AgentExtRpcCallback()])

    @classmethod
    def _device_to_port_id(cls, device):
        # REVISIT(rkukura): Consider calling into MechanismDrivers to
        # process device names, or having MechanismDrivers supply list
        # of device prefixes to strip.
        if device.startswith(TAP_DEVICE_PREFIX):
            return device[TAP_DEVICE_PREFIX_LENGTH:]
        else:
            return device

    @classmethod
    def get_port_from_device(cls, device):
        port_id = cls._device_to_port_id(device)
        port = db.get_port_and_sgs(port_id)
        if port:
            port['device'] = device
        return port

    def get_device_details(self, rpc_context, **kwargs):
        """Agent requests device details."""
        agent_id = kwargs.get('agent_id')
        device = kwargs.get('device')
        LOG.debug(_("Device %(device)s details requested by agent "
                    "%(agent_id)s"),
                  {'device': device, 'agent_id': agent_id})
        port_id = self._device_to_port_id(device)

        session = db_api.get_session()
        with session.begin(subtransactions=True):
            port = db.get_port(session, port_id)
            if not port:
                LOG.warning(_("Device %(device)s requested by agent "
                              "%(agent_id)s not found in database"),
                            {'device': device, 'agent_id': agent_id})
                return {'device': device}
            segments = db.get_network_segments(session, port.network_id)
            if not segments:
                LOG.warning(_("Device %(device)s requested by agent "
                              "%(agent_id)s has network %(network_id)s with "
                              "no segments"),
                            {'device': device,
                             'agent_id': agent_id,
                             'network_id': port.network_id})
                return {'device': device}
            #TODO(rkukura): Use/create port binding
            segment = segments[0]
            new_status = (q_const.PORT_STATUS_ACTIVE if port.admin_state_up
                          else q_const.PORT_STATUS_DOWN)
            if port.status != new_status:
                port.status = new_status
            entry = {'device': device,
                     'network_id': port.network_id,
                     'port_id': port.id,
                     'admin_state_up': port.admin_state_up,
                     'network_type': segment[api.NETWORK_TYPE],
                     'segmentation_id': segment[api.SEGMENTATION_ID],
                     'physical_network': segment[api.PHYSICAL_NETWORK]}
            LOG.debug(_("Returning: %s"), entry)
            return entry

    def update_device_down(self, rpc_context, **kwargs):
        """Device no longer exists on agent."""
        # TODO(garyk) - live migration and port status
        agent_id = kwargs.get('agent_id')
        device = kwargs.get('device')
        LOG.debug(_("Device %(device)s no longer exists at agent "
                    "%(agent_id)s"),
                  {'device': device, 'agent_id': agent_id})
        port_id = self._device_to_port_id(device)

        session = db_api.get_session()
        with session.begin(subtransactions=True):
            port = db.get_port(session, port_id)
            if not port:
                LOG.warning(_("Device %(device)s updated down by agent "
                              "%(agent_id)s not found in database"),
                            {'device': device, 'agent_id': agent_id})
                return {'device': device,
                        'exists': False}
            if port.status != q_const.PORT_STATUS_DOWN:
                port.status = q_const.PORT_STATUS_DOWN
            return {'device': device,
                    'exists': True}

    def update_device_up(self, rpc_context, **kwargs):
        """Device is up on agent."""
        agent_id = kwargs.get('agent_id')
        device = kwargs.get('device')
        LOG.debug(_("Device %(device)s up at agent %(agent_id)s"),
                  {'device': device, 'agent_id': agent_id})
        port_id = self._device_to_port_id(device)

        session = db_api.get_session()
        with session.begin(subtransactions=True):
            port = db.get_port(session, port_id)
            if not port:
                LOG.warning(_("Device %(device)s updated up by agent "
                              "%(agent_id)s not found in database"),
                            {'device': device, 'agent_id': agent_id})
            if port.status != q_const.PORT_STATUS_ACTIVE:
                port.status = q_const.PORT_STATUS_ACTIVE

class AgentNotifierApi(proxy.RpcProxy,
                       sg_rpc.SecurityGroupAgentRpcApiMixin,
                       type_tunnel.TunnelAgentRpcApiMixin):
    """Agent side of the openvswitch rpc API.

    API version history:
        1.0 - Initial version.
        1.1 - Added get_active_networks_info, create_dhcp_port,
              update_dhcp_port, and removed get_dhcp_port methods.

    """

    BASE_RPC_API_VERSION = '1.1'

    def __init__(self, topic):
        super(AgentNotifierApi, self).__init__(
            topic=topic, default_version=self.BASE_RPC_API_VERSION)
        self.topic_network_delete = topics.get_topic_name(topic,
                                                          topics.NETWORK,
                                                          topics.DELETE)
        self.topic_port_update = topics.get_topic_name(topic,
                                                       topics.PORT,
                                                       topics.UPDATE)

    def network_delete(self, context, network_id):
        self.fanout_cast(context,
                         self.make_msg('network_delete',
                                       network_id=network_id),
                         topic=self.topic_network_delete)

    def port_update(self, context, port, network_type, segmentation_id,
                    physical_network):
        self.fanout_cast(context,
                         self.make_msg('port_update',
                                       port=port,
                                       network_type=network_type,
                                       segmentation_id=segmentation_id,
                                       physical_network=physical_network),
                         topic=self.topic_port_update)
<|endoftext|>


<commit_before>import chess
import eco
import emails
import engine.store
import flask
import logs
import json
import os
import stats
import validate_email

from flask.ext import socketio

use_debug_server = False
allow_debug_routes = True

app = flask.Flask("chess")
rstore = engine.store.RedisStore()
sockapp = socketio.SocketIO(app)
eco_data = eco.load_default()

@app.route("/")
@logs.wrap
def index():
    return flask.render_template("index.html")

@app.route("/debug")
@logs.wrap
def debug():
    if not allow_debug_routes:
        abort(404)
    return flask.render_template("debug.html", games=rstore.all_games())

@app.route("/debug/create_game")
@logs.wrap
def debug_create_game():
    if not allow_debug_routes:
        abort(404)
    white_link, black_link = rstore.start_game("", "")
    white_url = _to_game_url(white_link)
    black_url = _to_game_url(black_link)
    return flask.render_template(
        "debug_create_game.html", white=white_url, black=black_url)

@app.route("/stats")
@logs.wrap
def global_stats():
    victories = stats.VictoryStats(rstore)
    plays = stats.PlayStats(rstore)
    return flask.render_template(
        "stats.html", victories=victories, plays=plays)

def _to_game_url(link):
    return "%sgame/%s" % (flask.request.url_root, link)

@app.route("/start", methods=["POST"])
@logs.wrap
def start():
    white_email = flask.request.form["white_email"]
    white_is_valid = validate_email.validate_email(white_email)
    black_email = flask.request.form["black_email"]
    black_is_valid = validate_email.validate_email(black_email)
    if not white_is_valid or not black_is_valid:
        print "Bad email addresses; %s (%s) %s (%s)" % (
            white_email, white_is_valid, black_email, black_is_valid)
        flask.abort(400)
    white_link, black_link = rstore.start_game(white_email, black_email)
    white_url = _to_game_url(white_link)
    black_url = _to_game_url(black_link)
    emails.send_welcome(white_email, white_url, black_email)
    emails.send_welcome(black_email, black_url, white_email)
    return flask.redirect(flask.url_for("getready"))

@app.route("/getready")
@logs.wrap
def getready():
    return flask.render_template("post_start.html")

def _make_move(game_link):
    color, game_id, game = rstore.game_from_link(game_link)
    if color != game.to_play:
        flask.abort(403)
    move_str = flask.request.form["move"]
    move = chess.parse_algebraic(
        game.current_board, color, move_str)
    rstore.move(game_id, move)
    if color == chess.white:
        opponent = chess.black
    else:
        opponent = chess.white
    player_email, _ = rstore.get_user(game_id, color)
    opponent_email, opponent_link = rstore.get_user(game_id, opponent)
    # Email addresses aren't set on debug games; don't try to send an email to
    # an empty address.
    if player_email and opponent_email:
        emails.send_move_email(
            opponent_email, _to_game_url(opponent_link), player_email, move_str)
    # Reload all the other players.
    sockapp.emit("reload", "", room=game_id)

@app.route("/game/<game_link>", methods=["GET", "POST"])
@logs.wrap
def game(game_link):
    if flask.request.method == "POST":
        _make_move(game_link)
        return "ok"
    color, game_id, game = rstore.game_from_link(game_link)
    access = chess.accessibility_map(game.current_board);
    opponent, _ = rstore.get_user(game_id, chess.opposite_color(color))
    if game.to_play is None:
        to_play = "nobody"
    else:
        to_play = chess.color_names[game.to_play]
    if game.termination is None:
        termination = "undefined"
        termination_msg = None
    else:
        termination = "'%s'" % game.termination
        if color == chess.white and game.termination == chess.white_victory:
            termination_msg = "Checkmate — you win!"
        elif color == chess.black and game.termination == chess.black_victory:
            termination_msg = "Checkmate — you win!"
        elif game.termination == chess.stalemate:
            termination_msg = "Stalemate."
        else:
            termination_msg = "Checkmate — you lose."
    stat_obj = stats.Stats(game, eco_data)
    return flask.render_template(
        "game.html",
        game=json.dumps(game.to_json_dict()),
        to_play=to_play,
        termination=termination,
        termination_msg=termination_msg,
        color_name=chess.color_names[color],
        player=color,
        summary=game.summary("\n"),
        accessibility=json.dumps(access),
        opponent=opponent,
        stats=stat_obj,
        )

@sockapp.on("join")
def on_join(data):
    # When clients join, we hook them into a room just for their game so that we
    # can force-reload them when the other player moves.
    link = data["link"]
    _, game_id, _ = rstore.game_from_link(link)
    socketio.join_room(game_id)

def run(api_keys):
    app.secret_key = api_keys.flask_session_key
    emails.set_mailgun_key(api_keys.mailgun_key)
    # Jinja does some funny shit here; just set the app directory to the
    # directory that web.py is in.
    app.root_path = os.path.abspath(os.path.dirname(__file__))
    if use_debug_server:
        app.run(host="0.0.0.0", debug=True)
    else:
        sockapp.run(app, log_file=logs.log_file)
<commit_msg>Unpush debug server.<commit_after>import chess
import eco
import emails
import engine.store
import flask
import logs
import json
import os
import stats
import validate_email

from flask.ext import socketio

use_debug_server = False
allow_debug_routes = False

app = flask.Flask("chess")
rstore = engine.store.RedisStore()
sockapp = socketio.SocketIO(app)
eco_data = eco.load_default()

@app.route("/")
@logs.wrap
def index():
    return flask.render_template("index.html")

@app.route("/debug")
@logs.wrap
def debug():
    if not allow_debug_routes:
        abort(404)
    return flask.render_template("debug.html", games=rstore.all_games())

@app.route("/debug/create_game")
@logs.wrap
def debug_create_game():
    if not allow_debug_routes:
        abort(404)
    white_link, black_link = rstore.start_game("", "")
    white_url = _to_game_url(white_link)
    black_url = _to_game_url(black_link)
    return flask.render_template(
        "debug_create_game.html", white=white_url, black=black_url)

@app.route("/stats")
@logs.wrap
def global_stats():
    victories = stats.VictoryStats(rstore)
    plays = stats.PlayStats(rstore)
    return flask.render_template(
        "stats.html", victories=victories, plays=plays)

def _to_game_url(link):
    return "%sgame/%s" % (flask.request.url_root, link)

@app.route("/start", methods=["POST"])
@logs.wrap
def start():
    white_email = flask.request.form["white_email"]
    white_is_valid = validate_email.validate_email(white_email)
    black_email = flask.request.form["black_email"]
    black_is_valid = validate_email.validate_email(black_email)
    if not white_is_valid or not black_is_valid:
        print "Bad email addresses; %s (%s) %s (%s)" % (
            white_email, white_is_valid, black_email, black_is_valid)
        flask.abort(400)
    white_link, black_link = rstore.start_game(white_email, black_email)
    white_url = _to_game_url(white_link)
    black_url = _to_game_url(black_link)
    emails.send_welcome(white_email, white_url, black_email)
    emails.send_welcome(black_email, black_url, white_email)
    return flask.redirect(flask.url_for("getready"))

@app.route("/getready")
@logs.wrap
def getready():
    return flask.render_template("post_start.html")

def _make_move(game_link):
    color, game_id, game = rstore.game_from_link(game_link)
    if color != game.to_play:
        flask.abort(403)
    move_str = flask.request.form["move"]
    move = chess.parse_algebraic(
        game.current_board, color, move_str)
    rstore.move(game_id, move)
    if color == chess.white:
        opponent = chess.black
    else:
        opponent = chess.white
    player_email, _ = rstore.get_user(game_id, color)
    opponent_email, opponent_link = rstore.get_user(game_id, opponent)
    # Email addresses aren't set on debug games; don't try to send an email to
    # an empty address.
    if player_email and opponent_email:
        emails.send_move_email(
            opponent_email, _to_game_url(opponent_link), player_email, move_str)
    # Reload all the other players.
    sockapp.emit("reload", "", room=game_id)

@app.route("/game/<game_link>", methods=["GET", "POST"])
@logs.wrap
def game(game_link):
    if flask.request.method == "POST":
        _make_move(game_link)
        return "ok"
    color, game_id, game = rstore.game_from_link(game_link)
    access = chess.accessibility_map(game.current_board);
    opponent, _ = rstore.get_user(game_id, chess.opposite_color(color))
    if game.to_play is None:
        to_play = "nobody"
    else:
        to_play = chess.color_names[game.to_play]
    if game.termination is None:
        termination = "undefined"
        termination_msg = None
    else:
        termination = "'%s'" % game.termination
        if color == chess.white and game.termination == chess.white_victory:
            termination_msg = "Checkmate — you win!"
        elif color == chess.black and game.termination == chess.black_victory:
            termination_msg = "Checkmate — you win!"
        elif game.termination == chess.stalemate:
            termination_msg = "Stalemate."
        else:
            termination_msg = "Checkmate — you lose."
    stat_obj = stats.Stats(game, eco_data)
    return flask.render_template(
        "game.html",
        game=json.dumps(game.to_json_dict()),
        to_play=to_play,
        termination=termination,
        termination_msg=termination_msg,
        color_name=chess.color_names[color],
        player=color,
        summary=game.summary("\n"),
        accessibility=json.dumps(access),
        opponent=opponent,
        stats=stat_obj,
        )

@sockapp.on("join")
def on_join(data):
    # When clients join, we hook them into a room just for their game so that we
    # can force-reload them when the other player moves.
    link = data["link"]
    _, game_id, _ = rstore.game_from_link(link)
    socketio.join_room(game_id)

def run(api_keys):
    app.secret_key = api_keys.flask_session_key
    emails.set_mailgun_key(api_keys.mailgun_key)
    # Jinja does some funny shit here; just set the app directory to the
    # directory that web.py is in.
    app.root_path = os.path.abspath(os.path.dirname(__file__))
    if use_debug_server:
        app.run(host="0.0.0.0", debug=True)
    else:
        sockapp.run(app, log_file=logs.log_file)
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 12-09-2017 16:10
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This file contains different models for the execution of subprocesses / external processes, e.g. via the command line
"""

import abc
import threading
# App imports
import config_manager
from .exceptions import ParallelRunnerException

# Execution of commands
class CommandLineRunnerFactory:
    pass

class ParallelManager:
    pass

class ParallelRunner(metaclass=abc.ABCMeta, threading.Thread):
    def __init__(self):
        super().__init__()
        self._logger = config_manager \
            .get_app_config_manager() \
            .get_logger_for("{}.{}".format(__name__, type(self).__name__))
        self.command = None
        self.timeout = None
        self._stdout = b' '
        self._stderr = b' '
        self._done = False
        self._shutdown = False

    @abc.abstractmethod
    def _run(self):
        ...

    def run(self):
        self._logger.debug("Parallel Runner ID '{}' --- START ---".format(threading.current_thread().getName()))
        self._run()

    def cancel(self):
        self._logger.debug("Parallel Runner ID '{}' --- CANCEL ---".format(threading.current_thread().getName()))
        self._shutdown = True
        self._stop()

    def wait(self):
        self._logger.debug("Parallel Runner ID '{}' --- WAIT ---".format(threading.current_thread().getName()))
        self.join()

    def get_stdout(self):
        # Never give it back until the runner is done with whatever it is doing
        if not self._done:
            raise ParallelRunnerException("Parallel Runner ID '{}' is NOT DONE doing its job, "
                                          "thus 'stdout' is NOT AVAILABLE".format(threading.current_thread().getName()))
        return self._stdout

    def get_stderr(self):
        pass

class CommandLineRunner(ParallelRunner):
    pass

class CommandLineRunnerAsThread(CommandLineRunner):
    pass

class CommandLineRunnerOnHpc(CommandLineRunner):
    pass
<commit_msg>stderr getter implementation<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 12-09-2017 16:10
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This file contains different models for the execution of subprocesses / external processes, e.g. via the command line
"""

import abc
import threading
# App imports
import config_manager
from .exceptions import ParallelRunnerException

# Execution of commands
class CommandLineRunnerFactory:
    pass

class ParallelManager:
    pass

class ParallelRunner(metaclass=abc.ABCMeta, threading.Thread):
    def __init__(self):
        super().__init__()
        self._logger = config_manager \
            .get_app_config_manager() \
            .get_logger_for("{}.{}".format(__name__, type(self).__name__))
        self.command = None
        self.timeout = None
        self._stdout = b' '
        self._stderr = b' '
        self._done = False
        self._shutdown = False

    @abc.abstractmethod
    def _run(self):
        ...

    def run(self):
        self._logger.debug("Parallel Runner ID '{}' --- START ---".format(threading.current_thread().getName()))
        self._run()

    def cancel(self):
        self._logger.debug("Parallel Runner ID '{}' --- CANCEL ---".format(threading.current_thread().getName()))
        self._shutdown = True
        self._stop()

    def wait(self):
        self._logger.debug("Parallel Runner ID '{}' --- WAIT ---".format(threading.current_thread().getName()))
        self.join()

    def get_stdout(self):
        # Never give it back until the runner is done with whatever it is doing
        if not self._done:
            raise ParallelRunnerException("Parallel Runner ID '{}' is NOT DONE doing its job, "
                                          "thus 'stdout' is NOT AVAILABLE".format(threading.current_thread().getName()))
        return self._stdout

    def get_stderr(self):
        # Never give it back until the runner is done with whatever it is doing
        if not self._done:
            raise ParallelRunnerException("Parallel Runner ID '{}' is NOT DONE doing its job, "
                                          "thus 'stderr' is NOT AVAILABLE".format(threading.current_thread().getName()))
        return self._stderr

class CommandLineRunner(ParallelRunner):
    pass

class CommandLineRunnerAsThread(CommandLineRunner):
    pass

class CommandLineRunnerOnHpc(CommandLineRunner):
    pass
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
scrape is a task to scrape rss resources
"""
# @author chengdujin
# @contact chengdujin@gmail.com
# @created Aug. 14, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
# sys.path.append('/home/work/newsman/newsman')
# sys.path.append('/home/users/jinyuan/newsman/newsman')
# sys.path.append('/home/ubuntu/newsman/newsman')
# sys.path.append('/home/jinyuan/Downloads/newsman/newsman')

from config.settings import Collection, db
from config.settings import logger
from parser import scraper
import Queue
import threading
import time

# CONSTANTS
from config.settings import FEED_REGISTRAR

class TimeoutQueue(Queue.Queue):

    """
    Synchronized Queue with timeout feature
    """

    def __init__(self):
        Queue.Queue.__init__(self)

    def join_with_timeout(self, timeout):
        self.all_tasks_done.acquire()
        try:
            endtime = time.time() + timeout
            while self.unfinished_tasks:
                remaining = endtime - time.time()
                if remaining <= 0.0:
                    raise Exception('[%s] unfinished items [%s]' %
                                    (str(self.unfinished_tasks), str(self.queue)))
                self.all_tasks_done.wait(remaining)
        finally:
            self.all_tasks_done.release()

class UpdateThread(threading.Thread):

    """
    Update news rss/twitter ...
    """

    def __init__(self, name):
        threading.Thread.__init__(self)
        self._name = name

    def run(self):
        while True:
            try:
                feed_id = queue.get()
                updated_feed = scraper.update(feed_id=feed_id)
                if updated_feed:
                    logger.error(
                        '%s: %s [%s] is successfully updated!' % (self._name, feed_id, updated_feed))
                else:
                    logger.error(
                        '%s: %s receives no update!' % (self._name, feed_id))
                queue.task_done()
            except Exception as k:
                logger.error(
                    '%s: [%s] receives no update but exception' % (self._name, str(k)))
                queue.task_done()

# synchronized queue that supports timeout
queue = TimeoutQueue()

def _update(feed_ids):
    """
    update links find in feeds
    """
    if not feed_ids:
        logger.error("No feed found!")
        return None

    # populate queue with feeds
    for feed_id in feed_ids:
        queue.put(feed_id)

    thread_limit = min(len(feed_ids) / 2, 25)
    for i in range(thread_limit):
        thread = UpdateThread('thread-%i' % i)
        thread.setDaemon(True)
        thread.start()

    try:
        # updating timeout set to 1.5 hours
        queue.join_with_timeout(5400)
    except Exception as k:
        logger.error(str(k))

def _read_feeds(language='en', country='US'):
    """
    read feed information from database feeds
    """
    try:
        db_feeds = Collection(db, FEED_REGISTRAR)
        items = db_feeds.find({'language': language, 'countries': country})
        if items:
            return [str(item['_id']) for item in items]
        else:
            logger.error("Cannot find any feeds for language %s!" % language)
            return None
    except Exception as k:
        logger.error(str(k))
        return None

def _scrape(language='en', country='US'):
    """
    update news from stored feeds
    """
    logger.error('############### scraping ###############')
    _update(_read_feeds(language, country))
    logger.error(
        "############### Feeds of %s_%s all got updated! ###############" %
        (language, country))

if __name__ == "__main__":
    language = sys.argv[1]
    country = sys.argv[2]
    _scrape(language, country)
<commit_msg>Modify output information<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
scrape is a task to scrape rss resources
"""
# @author chengdujin
# @contact chengdujin@gmail.com
# @created Aug. 14, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('/home/work/newsman/newsman')

from config.settings import Collection, db
from config.settings import logger
from parser import scraper
import Queue
import threading
import time

# CONSTANTS
from config.settings import FEED_REGISTRAR

class TimeoutQueue(Queue.Queue):

    """
    Synchronized Queue with timeout feature
    """

    def __init__(self):
        Queue.Queue.__init__(self)

    def join_with_timeout(self, timeout):
        self.all_tasks_done.acquire()
        try:
            endtime = time.time() + timeout
            while self.unfinished_tasks:
                remaining = endtime - time.time()
                if remaining <= 0.0:
                    raise Exception('[%s] unfinished items [%s]' %
                                    (str(self.unfinished_tasks), str(self.queue)))
                self.all_tasks_done.wait(remaining)
        finally:
            self.all_tasks_done.release()

class UpdateThread(threading.Thread):

    """
    Update news rss/twitter ...
    """

    def __init__(self, name):
        threading.Thread.__init__(self)
        self._name = name

    def run(self):
        while True:
            try:
                feed_id = queue.get()
                updated_feed = scraper.update(feed_id=feed_id)
                if updated_feed:
                    logger.warning(
                        '%s: %s [%s] is successfully updated!' % (self._name, feed_id, updated_feed))
                else:
                    logger.warning(
                        '%s: %s receives no update!' % (self._name, feed_id))
                queue.task_done()
            except Exception as k:
                logger.error(
                    '%s: [%s] receives no update but exception' % (self._name, str(k)))
                queue.task_done()

# synchronized queue that supports timeout
queue = TimeoutQueue()

def _update(feed_ids):
    """
    update links find in feeds
    """
    if not feed_ids:
        logger.error("No feed found!")
        return None

    # populate queue with feeds
    for feed_id in feed_ids:
        queue.put(feed_id)

    thread_limit = min(len(feed_ids) / 2, 25)
    for i in range(thread_limit):
        thread = UpdateThread('Thread-%i' % i)
        thread.setDaemon(True)
        thread.start()

    try:
        # updating timeout set to 1.5 hours
        queue.join_with_timeout(5400)
    except Exception as k:
        logger.error(str(k))

def _read_feeds(language='en', country='US'):
    """
    read feed information from database feeds
    """
    try:
        db_feeds = Collection(db, FEED_REGISTRAR)
        items = db_feeds.find({'language': language, 'countries': country})
        if items:
            return [str(item['_id']) for item in items]
        else:
            logger.error("Cannot find any feeds for language %s!" % language)
            return None
    except Exception as k:
        logger.error(str(k))
        return None

def _scrape(language='en', country='US'):
    """
    update news from stored feeds
    """
    logger.warning('############### Scraping begins ###############')
    _update(_read_feeds(language, country))
    logger.warning(
        "############### Feeds of %s_%s all got updated! ###############" %
        (language, country))

if __name__ == "__main__":
    language = sys.argv[1]
    country = sys.argv[2]
    _scrape(language, country)
<|endoftext|>


<commit_before>#!/usr/bin/env python3.6

import json
from flask import Flask, request, render_template, send_file
from flask_cors import CORS
import webcolors
import pygame
import pygame.camera

# file to write to
FILE_PATH = 'data.json'

APP = Flask(__name__,static_url_path='/static',
            template_folder='templates')
# allow cross origin requests, we need this because of jscolor
CORS(APP)

using_webcam = False

def get_image():
    """
    Uses the first camera to get an image and saves it to a file.
    """
    if using_webcam:
        return
    pygame.camera.init()
    c = pygame.camera.list_cameras()
    if c is None:
        # no cameras found
        return
    using_webcam = True
    # use the first camera, get the image and save it
    cam = pygame.camera.Camera(c[0], (1280, 720))
    cam.start()
    img = cam.get_image()
    pygame.image.save(img, "capture.jpg")
    cam.stop()
    using_webcam = False

@APP.route('/', methods=['GET'])
def index():
    return render_template('index.html')

# writes the data to the output file
def write(data):
    d = json.dumps(data)
    with open(FILE_PATH, 'w') as f:
        f.write(d)
        f.close()

@APP.route('/state', methods=['GET'])
def get_state():
    """
    GET /state

    Returns the current state of the lights.
    This will just return the current state of the data file, without checking that it is valid json.
    Not sure if there is any point in converting from json back into json again.
    """
    with open(FILE_PATH, 'r') as f:
        return f.readlines()

@APP.route('/image', methods=['GET'])
def get_webcam():
    """
    Gets an image from the webcam
    """
    get_image()
    return send_file('capture.jpg')

def validate_state(data: dict):
    """
    Checks that the provied object contains valid data

    Throws an assertionerror if invalid
    """
    assert isinstance(data, dict), "Data was not a dict"

    # validate colors
    assert data['color1'] is not None, "Color 1 was none."
    assert isinstance(data['color1'], (list, tuple)), "Color 1 was not a list of RGB"
    assert data['color2'] is not None, "Color 2 was none"
    assert isinstance(data['color2'], (list, tuple)), "Color 2 was not a list of RGB"

    # validate types for the values that we cast
    # this check is redundant when getting values from the form
    # but is not when we just get json from the user
    assert isinstance(data['random1'], bool), "Random 1 was not a bool"
    assert isinstance(data['random2'], bool), "random 2 was not a bool"
    assert isinstance(data['pattern'], (int, float)), "pattern was not an int"
    assert isinstance(data['length'], (int, float)), "length was not int or float"
    assert isinstance(data['delay'], (int, float)), "delay was not int or float"

    # check the bounds for some of the parameters
    assert data['length'] > 0, "length out of bounds"
    assert data['delay'] >= 0, "delay out of bounds"
    assert data['pattern'] >= 0, "pattern invalid"

@APP.route('/form', methods=['POST'])
def form_state():
    """
    POST /form
    Request body are url encoded form params.
    Color1 and Color2 may be hex strings that start with #

    Updates the state using the form parameters.
    """
    data = {}
    # get the state from request args
    # these can be part of the query string, but we don't really care if it's one or the other
    try:
        data['color1'] = webcolors.hex_to_rgb('#' + request.args['color1'])
        data['color2'] = webcolors.hex_to_rgb('#' + request.args['color2'])
        data['random1'] = bool(request.args['random1'])
        data['random2'] = bool(request.args['random2'])
        data['pattern'] = int(request.args['pattern'])
        data['length'] = int(request.args['length'])
        data['delay'] = int(request.args['delay'])
        validate_state(data)
    except Exception:
        # if we caught something, probably invalid request data
        return 'Bad!', 400
    write(data)
    return data, 200

@APP.route('/state', methods=['POST'])
def post_state():
    """
    POST /state

    Expects the following json from the request body

    {
    "color1": [ 255, 0, 0],
    "color2": [ 0, 255, 0]
    "random2": true,
    "random1": true,
    "length": 1,
    "delay": 1,
    "pattern": 1
    }

    Updates the current state from the json provided in the request body.
    """
    # from the request body, load some json
    try:
        data = request.get_json(force=True)
        validate_state(data)
    except Exception as e:
        print(e)
        # todo make this exception handling more specific
        # just return bad request, something was likely invalid
        return 'Bad! ' + str(type(e)) + str(e), 400
    # write this data to the state file if valid
    write(data)
    return json.dumps(data), 200

if __name__ == '__main__':
    # host the server on port 80
    # while we shouldn't require using the builtin server in production environment
    # I think that this is just fine for the use case that I require
    # we can just turn off debug mode in production
    APP.run(debug=False, host='0.0.0.0', port=80)
<commit_msg>fix broken image endpoint<commit_after>#!/usr/bin/env python3.6

import json
from flask import Flask, request, render_template, send_file
from flask_cors import CORS
import webcolors
import pygame
import pygame.camera

# file to write to
FILE_PATH = 'data.json'

APP = Flask(__name__,static_url_path='/static',
            template_folder='templates')
# allow cross origin requests, we need this because of jscolor
CORS(APP)

using_webcam = False

def get_image():
    """
    Uses the first camera to get an image and saves it to a file.
    """
    global using_webcam
    if using_webcam:
        return
    pygame.camera.init()
    c = pygame.camera.list_cameras()
    if c is None:
        # no cameras found
        return
    using_webcam = True
    # use the first camera, get the image and save it
    cam = pygame.camera.Camera(c[0], (1280, 720))
    cam.start()
    img = cam.get_image()
    pygame.image.save(img, "capture.jpg")
    cam.stop()
    using_webcam = False

@APP.route('/', methods=['GET'])
def index():
    return render_template('index.html')

# writes the data to the output file
def write(data):
    d = json.dumps(data)
    with open(FILE_PATH, 'w') as f:
        f.write(d)
        f.close()

@APP.route('/state', methods=['GET'])
def get_state():
    """
    GET /state

    Returns the current state of the lights.
    This will just return the current state of the data file, without checking that it is valid json.
    Not sure if there is any point in converting from json back into json again.
    """
    with open(FILE_PATH, 'r') as f:
        return f.readlines()

@APP.route('/image', methods=['GET'])
def get_webcam():
    """
    Gets an image from the webcam
    """
    get_image()
    return send_file('capture.jpg')

def validate_state(data: dict):
    """
    Checks that the provied object contains valid data

    Throws an assertionerror if invalid
    """
    assert isinstance(data, dict), "Data was not a dict"

    # validate colors
    assert data['color1'] is not None, "Color 1 was none."
    assert isinstance(data['color1'], (list, tuple)), "Color 1 was not a list of RGB"
    assert data['color2'] is not None, "Color 2 was none"
    assert isinstance(data['color2'], (list, tuple)), "Color 2 was not a list of RGB"

    # validate types for the values that we cast
    # this check is redundant when getting values from the form
    # but is not when we just get json from the user
    assert isinstance(data['random1'], bool), "Random 1 was not a bool"
    assert isinstance(data['random2'], bool), "random 2 was not a bool"
    assert isinstance(data['pattern'], (int, float)), "pattern was not an int"
    assert isinstance(data['length'], (int, float)), "length was not int or float"
    assert isinstance(data['delay'], (int, float)), "delay was not int or float"

    # check the bounds for some of the parameters
    assert data['length'] > 0, "length out of bounds"
    assert data['delay'] >= 0, "delay out of bounds"
    assert data['pattern'] >= 0, "pattern invalid"

@APP.route('/form', methods=['POST'])
def form_state():
    """
    POST /form
    Request body are url encoded form params.
    Color1 and Color2 may be hex strings that start with #

    Updates the state using the form parameters.
    """
    data = {}
    # get the state from request args
    # these can be part of the query string, but we don't really care if it's one or the other
    try:
        data['color1'] = webcolors.hex_to_rgb('#' + request.args['color1'])
        data['color2'] = webcolors.hex_to_rgb('#' + request.args['color2'])
        data['random1'] = bool(request.args['random1'])
        data['random2'] = bool(request.args['random2'])
        data['pattern'] = int(request.args['pattern'])
        data['length'] = int(request.args['length'])
        data['delay'] = int(request.args['delay'])
        validate_state(data)
    except Exception:
        # if we caught something, probably invalid request data
        return 'Bad!', 400
    write(data)
    return data, 200

@APP.route('/state', methods=['POST'])
def post_state():
    """
    POST /state

    Expects the following json from the request body

    {
    "color1": [ 255, 0, 0],
    "color2": [ 0, 255, 0]
    "random2": true,
    "random1": true,
    "length": 1,
    "delay": 1,
    "pattern": 1
    }

    Updates the current state from the json provided in the request body.
    """
    # from the request body, load some json
    try:
        data = request.get_json(force=True)
        validate_state(data)
    except Exception as e:
        print(e)
        # todo make this exception handling more specific
        # just return bad request, something was likely invalid
        return 'Bad! ' + str(type(e)) + str(e), 400
    # write this data to the state file if valid
    write(data)
    return json.dumps(data), 200

if __name__ == '__main__':
    # host the server on port 80
    # while we shouldn't require using the builtin server in production environment
    # I think that this is just fine for the use case that I require
    # we can just turn off debug mode in production
    APP.run(debug=False, host='0.0.0.0', port=80)
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 12-09-2017 16:10
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This file contains different models for the execution of subprocesses / external processes, e.g. via the command line
"""

import abc
import time
import random
import threading
import subprocess
# App imports
import config_manager
from .exceptions import ParallelRunnerException, CommandLineRunnerAsThreadException, NoMoreAliveRunnersException

# Abstract Factories
class CommandLineRunnerFactory:
    @staticmethod
    def get_command_line_runner():
        # This is the automatic selector between command line runners
        # TODO - This Factory is creating only local runners in the first iteration
        return CommandLineRunnerAsThread()

    # Having the following two methods allows the application to use a multithreaded command line runner in an HPC
    # system, e.g. in those situations where you don't want your command line runners to queue more jobs in an
    # sHPC environment
    @staticmethod
    def get_multithread_command_line_runner():
        return CommandLineRunnerAsThread()

    @staticmethod
    def get_hpc_command_line_runner():
        return CommandLineRunnerOnHpc()

class ParallelRunnerManagerFactory:
    @staticmethod
    def get_parallel_runner_manager():
        # TODO - This factory is creating only one kind of Parallel Runner Manager, more complex ones to come ...
        return ParallelRunnerManager()

# Parallel Runner Managers
class ParallelRunnerManager:
    # TODO - Include an 'auto_start' flag to start runners as they are added to the manager
    def __init__(self):
        self._logger = config_manager \
            .get_app_config_manager() \
            .get_logger_for("{}.{}".format(__name__, type(self).__name__))
        self.__runners = set()
        self.__alive_runners = set()
        self.__finished_runners = set()

    def add_runners(self, runners):
        self.__runners.union(set(runners))

    def add_runner(self, runner):
        self.__runners.add(runner)

    def start_runners(self):
        self._logger.debug("Starting #{} Runners".format(len(self.__runners)))
        for runner in self.__runners:
            runner.start()
            self.__alive_runners.add(runner)
        self._logger.debug("Runners started, clearing out the runners container")
        self.__runners.clear()

    def get_next_finished_runner(self):
        if not self.__alive_runners:
            raise NoMoreAliveRunnersException("No more runners left! They've all finished")
        runner_found = None
        counter = 1
        while True:
            self._logger.debug("Searching for the next finished runner among #{} runners, ROUND #{}"
                               .format(len(self.__alive_runners),
                                       counter))
            for runner in self.__alive_runners:
                if runner.is_done():
                    runner_found = runner
                    break
            if runner_found:
                self.__alive_runners.remove(runner_found)
                self.__finished_runners.add(runner_found)
                break
            # WARNING! - MAGIC NUMBER AHEAD!!!
            # We haven't found any runner on this round, let's wait a random amount of time before we try again
            time.sleep(random.randint(0, 10))
            counter += 1
        return runner_found

    def wait_all(self):
        self._logger.debug("Waiting for all #{} runners to finish".format(len(self.__alive_runners)))
        try:
            while True:
                self.get_next_finished_runner()
        except NoMoreAliveRunnersException as e:
            self._logger.debug("All runners are (should be) finished")

    def get_not_started_runners(self):
        return set(self.__runners)

    def get_alive_runners(self):
        return set(self.__alive_runners)

    def get_finished_runners(self):
        return set(self.__finished_runners)

# Parallel Runners
class ParallelRunner(threading.Thread, metaclass=abc.ABCMeta):
    # TODO - Refactor This, as responsibilities are a little bit mixed up
    def __init__(self):
        super().__init__()
        self._logger = config_manager \
            .get_app_config_manager() \
            .get_logger_for("{}.{}-{}".format(__name__, type(self).__name__, threading.current_thread().getName()))
        self._done = False
        self._error = False
        self._shutdown = False

    @abc.abstractmethod
    def _run(self):
        ...

    def run(self):
        self._logger.debug("--- START ---")
        try:
            self._run()
        except ParallelRunnerException as e:
            # This code is running on a separated thread, so this class, as top level 'client', must log the error for
            # the application
            self._logger.error("Parallel Runner failed ---> '{}'".format(e.value))
        finally:
            self._done = True

    def cancel(self):
        self._logger.debug("--- CANCEL ---")
        self._shutdown = True
        self._stop()

    def wait(self):
        self._logger.debug("--- WAIT ---")
        self.join()

    def is_done(self):
        return self._done

    def is_error(self):
        return self._error

# Execution of commands
class CommandLineRunner(ParallelRunner):
    def __init__(self):
        super().__init__()
        self._logger = config_manager \
            .get_app_config_manager() \
            .get_logger_for("{}.{}".format(__name__, type(self).__name__))
        self._stdout = b' '
        self._stderr = b' '
        self.command = None
        self.command_success = False
        self.command_return_code = 0
        self.timeout = None
        self.current_working_directory = None

    def get_stdout(self):
        # Never give it back until the runner is done with whatever it is doing
        if not self._done:
            raise ParallelRunnerException("Runner is NOT DONE doing its job, thus 'stdout' is NOT AVAILABLE")
        return self._stdout

    def get_stderr(self):
        # Never give it back until the runner is done with whatever it is doing
        if not self._done:
            raise ParallelRunnerException("Runner is NOT DONE doing its job, thus 'stderr' is NOT AVAILABLE")
        return self._stderr

class CommandLineRunnerAsThread(CommandLineRunner):
    def __init__(self):
        super().__init__()
        self._logger = config_manager \
            .get_app_config_manager() \
            .get_logger_for("{}.{}-{}".format(__name__, type(self).__name__, threading.current_thread().getName()))

    def _run(self):
        self._logger.debug("Preparing for running command '{}', "
                           "current working directory at '{}', "
                           "timeout '{}s'".format(self.command,
                                                  self.current_working_directory,
                                                  self.timeout))
        command_subprocess = subprocess.Popen(self.command,
                                              cwd=self.current_working_directory,
                                              stdout=subprocess.PIPE,
                                              stderr=subprocess.PIPE,
                                              shell=True)
        self._logger.debug("Communicating with subprocess for command '{}', "
                           "current working directory at '{}', "
                           "timeout '{}s'".format(self.command,
                                                  self.current_working_directory,
                                                  self.timeout))
        try:
            self._stdout, self._stderr = command_subprocess.communicate(timeout=self.timeout)
        except subprocess.TimeoutExpired as e:
            command_subprocess.kill()
            raise CommandLineRunnerAsThreadException("Communicating with subprocess for command '{}', "
                                                     "current working directory at '{}', "
                                                     "timeout '{}s'".format(self.command,
                                                                            self.current_working_directory,
                                                                            self.timeout)) from e
        self._logger.debug("Polling command '{}', "
                           "current working directory at '{}', "
                           "timeout '{}s'".format(self.command,
                                                  self.current_working_directory,
                                                  self.timeout))
        if command_subprocess.poll() and (command_subprocess.returncode != 0):
            self.command_return_code = command_subprocess.returncode
            raise CommandLineRunnerAsThreadException("ERROR - Return Code '{}' for command '{}', "
                                                     "current working directory at '{}', "
                                                     "timeout '{}s'".format(command_subprocess.returncode,
                                                                            self.command,
                                                                            self.current_working_directory,
                                                                            self.timeout))
        # I don't think I really need this flag over here, as any other situation would throw an exception but, it looks
        # good and, keep into account that whatever you do with a possible exception is independent than whatever you
        # can express by the combination of flags '_done' and 'command_success'. This allows you to react by capturing
        # the exception (that also provides the return code for the command) but still keep the information that the
        # process finished Ok, but the command failed, and you also have the stdout and stderr content for further
        # analysis
        self.command_success = True

class CommandLineRunnerOnHpc(CommandLineRunner):
    def __init__(self):
        super().__init__()
        self._logger = config_manager \
            .get_app_config_manager() \
            .get_logger_for("{}.{}-{}".format(__name__, type(self).__name__, threading.current_thread().getName()))

    def _run(self):
        pass

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<commit_msg>set the error flag upon exception running the parallel kernel<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 12-09-2017 16:10
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This file contains different models for the execution of subprocesses / external processes, e.g. via the command line
"""

import abc
import time
import random
import threading
import subprocess
# App imports
import config_manager
from .exceptions import ParallelRunnerException, CommandLineRunnerAsThreadException, NoMoreAliveRunnersException

# Abstract Factories
class CommandLineRunnerFactory:
    @staticmethod
    def get_command_line_runner():
        # This is the automatic selector between command line runners
        # TODO - This Factory is creating only local runners in the first iteration
        return CommandLineRunnerAsThread()

    # Having the following two methods allows the application to use a multithreaded command line runner in an HPC
    # system, e.g. in those situations where you don't want your command line runners to queue more jobs in an
    # sHPC environment
    @staticmethod
    def get_multithread_command_line_runner():
        return CommandLineRunnerAsThread()

    @staticmethod
    def get_hpc_command_line_runner():
        return CommandLineRunnerOnHpc()

class ParallelRunnerManagerFactory:
    @staticmethod
    def get_parallel_runner_manager():
        # TODO - This factory is creating only one kind of Parallel Runner Manager, more complex ones to come ...
        return ParallelRunnerManager()

# Parallel Runner Managers
class ParallelRunnerManager:
    # TODO - Include an 'auto_start' flag to start runners as they are added to the manager
    def __init__(self):
        self._logger = config_manager \
            .get_app_config_manager() \
            .get_logger_for("{}.{}".format(__name__, type(self).__name__))
        self.__runners = set()
        self.__alive_runners = set()
        self.__finished_runners = set()

    def add_runners(self, runners):
        self.__runners.union(set(runners))

    def add_runner(self, runner):
        self.__runners.add(runner)

    def start_runners(self):
        self._logger.debug("Starting #{} Runners".format(len(self.__runners)))
        for runner in self.__runners:
            runner.start()
            self.__alive_runners.add(runner)
        self._logger.debug("Runners started, clearing out the runners container")
        self.__runners.clear()

    def get_next_finished_runner(self):
        if not self.__alive_runners:
            raise NoMoreAliveRunnersException("No more runners left! They've all finished")
        runner_found = None
        counter = 1
        while True:
            self._logger.debug("Searching for the next finished runner among #{} runners, ROUND #{}"
                               .format(len(self.__alive_runners),
                                       counter))
            for runner in self.__alive_runners:
                if runner.is_done():
                    runner_found = runner
                    break
            if runner_found:
                self.__alive_runners.remove(runner_found)
                self.__finished_runners.add(runner_found)
                break
            # WARNING! - MAGIC NUMBER AHEAD!!!
            # We haven't found any runner on this round, let's wait a random amount of time before we try again
            time.sleep(random.randint(0, 10))
            counter += 1
        return runner_found

    def wait_all(self):
        self._logger.debug("Waiting for all #{} runners to finish".format(len(self.__alive_runners)))
        try:
            while True:
                self.get_next_finished_runner()
        except NoMoreAliveRunnersException as e:
            self._logger.debug("All runners are (should be) finished")

    def get_not_started_runners(self):
        return set(self.__runners)

    def get_alive_runners(self):
        return set(self.__alive_runners)

    def get_finished_runners(self):
        return set(self.__finished_runners)

# Parallel Runners
class ParallelRunner(threading.Thread, metaclass=abc.ABCMeta):
    # TODO - Refactor This, as responsibilities are a little bit mixed up
    def __init__(self):
        super().__init__()
        self._logger = config_manager \
            .get_app_config_manager() \
            .get_logger_for("{}.{}-{}".format(__name__, type(self).__name__, threading.current_thread().getName()))
        self._done = False
        self._error = False
        self._shutdown = False

    @abc.abstractmethod
    def _run(self):
        ...

    def run(self):
        self._logger.debug("--- START ---")
        try:
            self._run()
        except ParallelRunnerException as e:
            # This code is running on a separated thread, so this class, as top level 'client', must log the error for
            # the application
            self._logger.error("Parallel Runner failed ---> '{}'".format(e.value))
            self._error = True
        finally:
            self._done = True

    def cancel(self):
        self._logger.debug("--- CANCEL ---")
        self._shutdown = True
        self._stop()

    def wait(self):
        self._logger.debug("--- WAIT ---")
        self.join()

    def is_done(self):
        return self._done

    def is_error(self):
        return self._error

# Execution of commands
class CommandLineRunner(ParallelRunner):
    def __init__(self):
        super().__init__()
        self._logger = config_manager \
            .get_app_config_manager() \
            .get_logger_for("{}.{}".format(__name__, type(self).__name__))
        self._stdout = b' '
        self._stderr = b' '
        self.command = None
        self.command_success = False
        self.command_return_code = 0
        self.timeout = None
        self.current_working_directory = None

    def get_stdout(self):
        # Never give it back until the runner is done with whatever it is doing
        if not self._done:
            raise ParallelRunnerException("Runner is NOT DONE doing its job, thus 'stdout' is NOT AVAILABLE")
        return self._stdout

    def get_stderr(self):
        # Never give it back until the runner is done with whatever it is doing
        if not self._done:
            raise ParallelRunnerException("Runner is NOT DONE doing its job, thus 'stderr' is NOT AVAILABLE")
        return self._stderr

class CommandLineRunnerAsThread(CommandLineRunner):
    def __init__(self):
        super().__init__()
        self._logger = config_manager \
            .get_app_config_manager() \
            .get_logger_for("{}.{}-{}".format(__name__, type(self).__name__, threading.current_thread().getName()))

    def _run(self):
        self._logger.debug("Preparing for running command '{}', "
                           "current working directory at '{}', "
                           "timeout '{}s'".format(self.command,
                                                  self.current_working_directory,
                                                  self.timeout))
        command_subprocess = subprocess.Popen(self.command,
                                              cwd=self.current_working_directory,
                                              stdout=subprocess.PIPE,
                                              stderr=subprocess.PIPE,
                                              shell=True)
        self._logger.debug("Communicating with subprocess for command '{}', "
                           "current working directory at '{}', "
                           "timeout '{}s'".format(self.command,
                                                  self.current_working_directory,
                                                  self.timeout))
        try:
            self._stdout, self._stderr = command_subprocess.communicate(timeout=self.timeout)
        except subprocess.TimeoutExpired as e:
            command_subprocess.kill()
            raise CommandLineRunnerAsThreadException("Communicating with subprocess for command '{}', "
                                                     "current working directory at '{}', "
                                                     "timeout '{}s'".format(self.command,
                                                                            self.current_working_directory,
                                                                            self.timeout)) from e
        self._logger.debug("Polling command '{}', "
                           "current working directory at '{}', "
                           "timeout '{}s'".format(self.command,
                                                  self.current_working_directory,
                                                  self.timeout))
        if command_subprocess.poll() and (command_subprocess.returncode != 0):
            self.command_return_code = command_subprocess.returncode
            raise CommandLineRunnerAsThreadException("ERROR - Return Code '{}' for command '{}', "
                                                     "current working directory at '{}', "
                                                     "timeout '{}s'".format(command_subprocess.returncode,
                                                                            self.command,
                                                                            self.current_working_directory,
                                                                            self.timeout))
        # I don't think I really need this flag over here, as any other situation would throw an exception but, it looks
        # good and, keep into account that whatever you do with a possible exception is independent than whatever you
        # can express by the combination of flags '_done' and 'command_success'. This allows you to react by capturing
        # the exception (that also provides the return code for the command) but still keep the information that the
        # process finished Ok, but the command failed, and you also have the stdout and stderr content for further
        # analysis
        self.command_success = True

class CommandLineRunnerOnHpc(CommandLineRunner):
    def __init__(self):
        super().__init__()
        self._logger = config_manager \
            .get_app_config_manager() \
            .get_logger_for("{}.{}-{}".format(__name__, type(self).__name__, threading.current_thread().getName()))

    def _run(self):
        pass

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<|endoftext|>


<commit_before>#!/usr/bin/env python

from random import randint, choice
from datetime import datetime
import sys
import argparse

country_code = 234 #Nigeria
max_characters = 13 #Maximum characters of GSM numbers you want

pad_number = randint(0,9)
number_range = {

        "mtn": [ 703, 803, 806, 813, 814, 816, 903],
        "airtel": [ 701, 802, 808, 812, 902],
        "glo": [ 705, 805, 807, 811, 815, 905],
        "etisalat": [ 809, 817, 818, 909]
}

def generate(operator, count):

    output_file = operator.upper()+"_"+str(datetime.now().strftime('%d%m%Y%M%S'))+".csv"
    if operator.lower() in number_range.keys():
        try:
            assert count > 0
        except AssertionError:
            print "Count must be greater than 0"
            sys.exit()
        while count > 1:
            with open(output_file, "a") as out_file:
                randval = randint(00000000, 99999999)
                mobile_number = str(country_code)+str(choice(number_range[operator]))+str(randval)
                if len(mobile_number) < max_characters:
                    count_difference = int(max_characters) - len(mobile_number)
                    full_mobile_number = (lambda x: x + count_difference*str(randint(0,9)) )(mobile_number)
                    out_file.write(full_mobile_number[:max_characters]+"\n")
                    count -= 1
    else:
        print "Telco Operator profile not found"
        sys.exit()

parser = argparse.ArgumentParser(description='Generates GSM Mobile Numbers')
parser.add_argument('operator', type=str, help="Short name for operator e.g MTN, Glo")
parser.add_argument('count', type=int, help="Quantity of MSISDN needed")
args = parser.parse_args()

if __name__=='__main__':
    generate(args.operator, args.count)

<commit_msg>Restructure project<commit_after><|endoftext|>


<commit_before>import collections
import json
import random
import re

def num(s):
    try:
        return int(s)
    except ValueError:
        return float(s)

# load a json file into a dictionary
def load_json(filename):
    f = json.load(open(filename), object_pairs_hook=collections.OrderedDict)
    return f

# generate an entity from a json profile
def generate_entity(data):
    # generate a blank entity
    entity = {}
    # for each key in the data profile
    for key, options in data["profile"].items():
        # generate a value for the entity according to the list of options
        entity[key] = next(choose_options(data["resources"], options, entity, 1))

    return entity

def choose_options(res, options, entity, n):
    # keep a list of options for which the entity fulfills all requirements
    valid_options = []
    # for each option
    for opt in options:
        # check if entity fulfills all requirements
        if check_reqs(opt, entity):
            # add the option to the valid list
            valid_options.append([parse_weight({}, opt['wt']), opt['val']])

    # get a generator that returns the chosen options
    chosen = weighted_selection(valid_options, n)
    # yield each chosen option
    while n:
        # replace commands inside the value of the next chosen
        value = parse_commands(res, entity, next(chosen))

        # return the completed value
        yield value
        n -= 1

# parses the expression that represents the weight of an object, and evaluates the expression to return an integer
def parse_weight(context, weight):
    # replace selection commands in the expression
    # get list of all matches
    matches = re.findall("\$(.*?)\$", weight)
    # unique list
    match_set = set(matches)
    # for each unique match
    for match in match_set:
        # replace command with its replacement value
        try:
            weight = weight.replace('$'+match+'$', context[match])
        except KeyError:
            weight = weight.replace('$'+match+'$', "1")

    # evaluate the expression with constants
    # THIS IS SUPER UNSAFE, WILL EXECUTE ARBITRARY CODE
    return eval(weight)

# weighted selection without replacement
def weighted_selection(items, n):
    total_weight = float(sum(wt for wt, val in items))
    while n:
        index = 0
        running_weight = items[0][0]
        rand_num = random.random() * total_weight
        while rand_num > running_weight:
            index += 1
            running_weight += items[index][0]
        wt, val = items.pop(index)
        total_weight -= wt
        yield val

# removes all command syntax from a string, replacing it with a valid value for that command
def parse_commands(res, entity, string):
    string = parse_selection(res, entity, string)
    string = parse_numeric(string)

    return string

# selection command: $...$
# returns a string with instances of a selection command replaced by a value selected by the command
# res is the list of resources that the selection command will pull from
# entity is the entity being generated as it currently exists, used for checking against requirements
def parse_selection(res, entity, string):
    # get list of all matches
    matches = re.findall("\$(.*?)\$", string)
    # unique list
    match_set = set(matches)
    # for each unique match
    for match in match_set:
        # get number of occurrences in original string
        occurences = matches.count(match)
        # get replacement values
        replacements = choose_options(res, res[match], entity, occurences)
        # for each occurence
        for i in range(occurences):
            # replace command with a unique replacement value
            string = string.replace('$'+match+'$', next(replacements), 1)

    return string

# numeric generation command: [...]
# returns a string with instances of a numeric generation command replaces with a number generated from the command
# string is the string that will be parsed
def parse_numeric(string):
    while True:
        # break when no more replacements are needed
        matches = re.search("\[([0-9]*)\-([0-9]*)\]", string)
        if matches is None:
            break
        # generate replacement value
        replacement = str(random.randint(int(matches.group(1)), int(matches.group(2))))
        # replace command with its replacement value
        string = string.replace(matches.group(0), replacement)

    return string

# checks an entity to ensure that it follows all requirements in options
# options is the option that entity is being checked against to see if it meets the requirements
# entity is the entity that is being checked against the requirements of option
def check_reqs(option, entity):

    # for each key in requirements
    for key, values in option["req"].items():
        if not check_attr(values, entity, key):
            # return that one or more requirements were not met
            return False

    # if no attribute requirements went unfulfilled, return that all requirements were met
    return True

# checks whether an attribute on entity matches any one of the possible values for the requirement
# vals is the list of possible values that entity[attr] can have
# entity is the entity being checked
# attr is the particular attribute being checked on the entity
def check_attr(vals, entity, attr):
    # if it meets one of the requirements, return True
    for value in vals:
        if check_req(value, entity, attr):
            return True

    # if it met none of the requirements, return False
    return False

# checks whether exactly one possible value of a requirement matches the corresponding attribute attr on entity
# req is the value that is being compared to entity[attr]
# entity is the entity that is attempting to fulfill the requirement
# attr is the attribute of entity that must match the value of req
def check_req(req, entity, attr):

    compare_nums = re.match("\[(<|>|<=|>=|=)([0-9]+)\]", req)
    # if the value is a numeric comparison
    if not compare_nums is None:
        try:
            num(entity[attr])
        except ValueError:
            print("Attempted to do a numeric comparison on a non-numeric entity attribute: "+entity.type+"["+attr+"]")

        return compare(compare_nums.group(1), entity[attr], compare_nums.group(2))
    # if the requirement is a straight text comparison
    else:
        if entity[attr] == req:
            return True
        else:
            return False

# takes 3 strings
# operator is a string containing the operator to use for comparison (one of =, <, <=, >, >=)
# a and b are the values that will be compared
# both a and b must be numeric
def compare(operator, a, b):
    if operator == "=":
        return num(a) == num(b)
    elif operator == "<":
        return num(a) < num(b)
    elif operator == "<=":
        return num(a) <= num(b)
    elif operator == ">":
        return num(a) > num(b)
    elif operator == ">=":
        return num(a) >= num(b)
<commit_msg>refactored code regarding option selection during entity generation<commit_after>import collections
import json
import random
import re

def num(s):
    try:
        return int(s)
    except ValueError:
        return float(s)

# load a json file into a dictionary
def load_json(filename):
    f = json.load(open(filename), object_pairs_hook=collections.OrderedDict)
    return f

# generate an entity from a json profile
def generate_entity(data):
    # generate a blank entity
    entity = {}
    # for each key in the data profile
    for key, options in data["profile"].items():
        # generate a value for the entity according to the list of options
        value = next(choose_options(options, entity, 1))
        # parses commands out of the value generated
        entity[key] = parse_commands(data['resources'], entity, value)

    return entity

# returns a generator which yields a maximum of n items from options for which entity fulfills all requirements
def choose_options(options, entity, n):
    # keep a list of options for which the entity fulfills all requirements
    valid_options = []
    # for each option
    for opt in options:
        # check if entity fulfills all requirements
        if check_reqs(opt, entity):
            # add the option to the valid list
            valid_options.append([parse_weight({}, opt['wt']), opt['val']])

    # get a generator that returns the chosen options
    chosen = weighted_selection(valid_options, n)
    # yield each chosen option
    while n:
        # return the completed value
        yield next(chosen)
        n -= 1

# parses the expression that represents the weight of an object, and evaluates the expression to return an integer
def parse_weight(context, weight):
    # replace selection commands in the expression
    # get list of all matches
    matches = re.findall("\$(.*?)\$", weight)
    # unique list
    match_set = set(matches)
    # for each unique match
    for match in match_set:
        # replace command with its replacement value
        try:
            weight = weight.replace('$'+match+'$', context[match])
        except KeyError:
            weight = weight.replace('$'+match+'$', "1")

    # evaluate the expression with constants
    # THIS IS SUPER UNSAFE, WILL EXECUTE ARBITRARY CODE
    return eval(weight)

# weighted selection without replacement
def weighted_selection(items, n):
    total_weight = float(sum(wt for wt, val in items))
    while n:
        index = 0
        running_weight = items[0][0]
        rand_num = random.random() * total_weight
        while rand_num > running_weight:
            index += 1
            running_weight += items[index][0]
        wt, val = items.pop(index)
        total_weight -= wt
        yield val

# removes all command syntax from a string, replacing it with a valid value for that command
def parse_commands(res, entity, string):
    oldstring = ""
    while not string == oldstring:
        oldstring = string
        string = parse_selection(res, entity, string)
        string = parse_numeric(string)

    return string

# selection command: $...$
# returns a string with instances of a selection command replaced by a value selected by the command
# res is the list of resources that the selection command will pull from
# entity is the entity being generated as it currently exists, used for checking against requirements
def parse_selection(res, entity, string):
    # get list of all matches
    matches = re.findall("\$(.*?)\$", string)
    # unique list
    match_set = set(matches)
    # for each unique match
    for match in match_set:
        # get number of occurrences in original string
        occurences = matches.count(match)
        # get replacement values
        replacements = choose_options(res[match], entity, occurences)
        # for each occurence
        for i in range(occurences):
            # replace command with a unique replacement value
            string = string.replace('$'+match+'$', next(replacements), 1)

    return string

# numeric generation command: [...]
# returns a string with instances of a numeric generation command replaces with a number generated from the command
# string is the string that will be parsed
def parse_numeric(string):
    while True:
        # break when no more replacements are needed
        matches = re.search("\[([0-9]*)\-([0-9]*)\]", string)
        if matches is None:
            break
        # generate replacement value
        replacement = str(random.randint(int(matches.group(1)), int(matches.group(2))))
        # replace command with its replacement value
        string = string.replace(matches.group(0), replacement)

    return string

# checks an entity to ensure that it follows all requirements in options
# options is the option that entity is being checked against to see if it meets the requirements
# entity is the entity that is being checked against the requirements of option
def check_reqs(option, entity):

    # for each key in requirements
    for key, values in option["req"].items():
        if not check_attr(values, entity, key):
            # return that one or more requirements were not met
            return False

    # if no attribute requirements went unfulfilled, return that all requirements were met
    return True

# checks whether an attribute on entity matches any one of the possible values for the requirement
# vals is the list of possible values that entity[attr] can have
# entity is the entity being checked
# attr is the particular attribute being checked on the entity
def check_attr(vals, entity, attr):
    # if it meets one of the requirements, return True
    for value in vals:
        if check_req(value, entity, attr):
            return True

    # if it met none of the requirements, return False
    return False

# checks whether exactly one possible value of a requirement matches the corresponding attribute attr on entity
# req is the value that is being compared to entity[attr]
# entity is the entity that is attempting to fulfill the requirement
# attr is the attribute of entity that must match the value of req
def check_req(req, entity, attr):

    compare_nums = re.match("\[(<|>|<=|>=|=)([0-9]+)\]", req)
    # if the value is a numeric comparison
    if not compare_nums is None:
        try:
            num(entity[attr])
        except ValueError:
            print("Attempted to do a numeric comparison on a non-numeric entity attribute: "+entity.type+"["+attr+"]")

        return compare(compare_nums.group(1), entity[attr], compare_nums.group(2))
    # if the requirement is a straight text comparison
    else:
        if entity[attr] == req:
            return True
        else:
            return False

# takes 3 strings
# operator is a string containing the operator to use for comparison (one of =, <, <=, >, >=)
# a and b are the values that will be compared
# both a and b must be numeric
def compare(operator, a, b):
    if operator == "=":
        return num(a) == num(b)
    elif operator == "<":
        return num(a) < num(b)
    elif operator == "<=":
        return num(a) <= num(b)
    elif operator == ">":
        return num(a) > num(b)
    elif operator == ">=":
        return num(a) >= num(b)
<|endoftext|>