<commit_before>from pyramid_mailer.interfaces import IMailer

from apex.i18n import MessageFactory

from sqlalchemy import engine_from_config

from pyramid.interfaces import IAuthenticationPolicy
from pyramid.interfaces import IAuthorizationPolicy
from pyramid.interfaces import ISessionFactory
from pyramid.session import UnencryptedCookieSessionFactoryConfig
from pyramid.settings import asbool
from pyramid.authentication import AuthTktAuthenticationPolicy
from pyramid.authorization import ACLAuthorizationPolicy
from pyramid.exceptions import Forbidden

from apex.exceptions import ApexAuthSecret
from apex.exceptions import ApexSessionSecret
from apex.interfaces import IApex
from apex.interfaces import ApexImplementation
from apex.lib.libapex import groupfinder
from apex.lib.libapex import RequestFactory
from apex.lib.libapex import RootFactory
from apex.models import initialize_sql
from apex.views import apex_callback
from apex.views import activate
from apex.views import change_password
from apex.views import edit
from apex.views import login
from apex.views import logout
from apex.views import forgot_password
from apex.views import forbidden
from apex.views import openid_required
from apex.views import register
from apex.views import reset_password

"""
    Allows flash messages to be called as:
::
    from apex import flash

"""

from apex.lib.flash import flash

def includeme(config):
    settings = config.registry.settings

    initialize_sql(engine_from_config(settings, 'sqlalchemy.'), settings)

    config.registry.registerUtility(ApexImplementation, IApex)
    config.add_translation_dirs('apex:locale/')

    if not config.registry.queryUtility(ISessionFactory):
        if not settings.has_key('apex.session_secret'):
            raise ApexSessionSecret()

        config.set_session_factory( \
               UnencryptedCookieSessionFactoryConfig( \
               settings.get('apex.session_secret')))

    if not config.registry.queryUtility(IAuthorizationPolicy):
        authz_policy = ACLAuthorizationPolicy()
        config.set_authorization_policy(authz_policy)

    if not config.registry.queryUtility(IAuthenticationPolicy):
        if not settings.has_key('apex.auth_secret'):
            raise ApexAuthSecret()
        authn_policy = AuthTktAuthenticationPolicy( \
                       settings.get('apex.auth_secret'), \
                       callback=groupfinder)
        config.set_authentication_policy(authn_policy)

    cache = RootFactory.__acl__
    config.set_root_factory(RootFactory)

    use_request_factory = asbool(settings.get('apex.use_request_factory', True))
    if use_request_factory:
        config.set_request_factory(RequestFactory)

    if not config.registry.queryUtility(IMailer):
        config.include('pyramid_mailer')

    if not settings.get('mako.directories'):
        config.add_settings({'mako.directories': ['apex:templates']})

    config.add_subscriber('apex.lib.subscribers.csrf_validation',
                          'pyramid.events.ContextFound')
    config.add_subscriber('apex.lib.subscribers.add_renderer_globals',
                          'pyramid.events.BeforeRender')

    config.add_static_view('apex/static', 'apex:static')

    config.add_view(forbidden, context=Forbidden)

    render_template = settings['apex.apex_render_template'
                              ] = settings.get('apex.apex_template',
                                               'apex:templates/apex_template.mako')

    config.add_route('apex_login', '/login')
    config.add_view(login, route_name='apex_login',
                    renderer=render_template)

    config.add_route('apex_logout', '/logout')
    config.add_view(logout, route_name='apex_logout',
                    renderer=render_template)

    config.add_route('apex_register', '/register')
    config.add_view(register, route_name='apex_register',
                    renderer=render_template)

    config.add_route('apex_password', '/password')
    config.add_view(change_password, route_name='apex_password',
                    renderer=render_template, permission='authenticated')

    config.add_route('apex_forgot', '/forgot')
    config.add_view(forgot_password, route_name='apex_forgot',
                    renderer=render_template)

    config.add_route('apex_reset', '/reset/:user_id/:hmac')
    config.add_view(reset_password, route_name='apex_reset',
                    renderer=render_template)

    config.add_route('apex_activate', '/activate/:user_id/:hmac')
    config.add_view(activate, route_name='apex_activate',
                    renderer=render_template)

    config.add_route('apex_callback', '/apex_callback')
    config.add_view(apex_callback, route_name='apex_callback')

    config.add_route('apex_openid_required', '/openid_required')
    config.add_view(openid_required, route_name= \
                    'apex_openid_required', \
                    renderer=render_template)

    if settings.has_key('apex.auth_profile'):
        use_edit = asbool(settings.get('apex.use_apex_edit', False))
        if use_edit:
            config.add_route('apex_edit', '/edit')
            config.add_view(edit, route_name='apex_edit', \
                            renderer=render_template, \
                            permission='authenticated')
<commit_msg>Added NO_PERMISSION_REQUIRED to routes that will not need permission, this allows a user to set a default_permission<commit_after>from pyramid_mailer.interfaces import IMailer

from apex.i18n import MessageFactory

from sqlalchemy import engine_from_config

from pyramid.interfaces import IAuthenticationPolicy
from pyramid.interfaces import IAuthorizationPolicy
from pyramid.interfaces import ISessionFactory
from pyramid.security import NO_PERMISSION_REQUIRED
from pyramid.session import UnencryptedCookieSessionFactoryConfig
from pyramid.settings import asbool
from pyramid.authentication import AuthTktAuthenticationPolicy
from pyramid.authorization import ACLAuthorizationPolicy
from pyramid.exceptions import Forbidden

from apex.exceptions import ApexAuthSecret
from apex.exceptions import ApexSessionSecret
from apex.interfaces import IApex
from apex.interfaces import ApexImplementation
from apex.lib.libapex import groupfinder
from apex.lib.libapex import RequestFactory
from apex.lib.libapex import RootFactory
from apex.models import initialize_sql
from apex.views import apex_callback
from apex.views import activate
from apex.views import change_password
from apex.views import edit
from apex.views import login
from apex.views import logout
from apex.views import forgot_password
from apex.views import forbidden
from apex.views import openid_required
from apex.views import register
from apex.views import reset_password

"""
    Allows flash messages to be called as:
::
    from apex import flash

"""

from apex.lib.flash import flash

def includeme(config):
    settings = config.registry.settings

    initialize_sql(engine_from_config(settings, 'sqlalchemy.'), settings)

    config.registry.registerUtility(ApexImplementation, IApex)
    config.add_translation_dirs('apex:locale/')

    if not config.registry.queryUtility(ISessionFactory):
        if not settings.has_key('apex.session_secret'):
            raise ApexSessionSecret()

        config.set_session_factory( \
               UnencryptedCookieSessionFactoryConfig( \
               settings.get('apex.session_secret')))

    if not config.registry.queryUtility(IAuthorizationPolicy):
        authz_policy = ACLAuthorizationPolicy()
        config.set_authorization_policy(authz_policy)

    if not config.registry.queryUtility(IAuthenticationPolicy):
        if not settings.has_key('apex.auth_secret'):
            raise ApexAuthSecret()
        authn_policy = AuthTktAuthenticationPolicy( \
                       settings.get('apex.auth_secret'), \
                       callback=groupfinder)
        config.set_authentication_policy(authn_policy)

    cache = RootFactory.__acl__
    config.set_root_factory(RootFactory)

    use_request_factory = asbool(settings.get('apex.use_request_factory', True))
    if use_request_factory:
        config.set_request_factory(RequestFactory)

    if not config.registry.queryUtility(IMailer):
        config.include('pyramid_mailer')

    if not settings.get('mako.directories'):
        config.add_settings({'mako.directories': ['apex:templates']})

    config.add_subscriber('apex.lib.subscribers.csrf_validation',
                          'pyramid.events.ContextFound')
    config.add_subscriber('apex.lib.subscribers.add_renderer_globals',
                          'pyramid.events.BeforeRender')

    config.add_static_view('apex/static', 'apex:static')

    config.add_view(forbidden, context=Forbidden, permission=NO_PERMISSION_REQUIRED)

    render_template = settings['apex.apex_render_template'
                              ] = settings.get('apex.apex_template',
                                               'apex:templates/apex_template.mako')

    config.add_route('apex_login', '/login')
    config.add_view(login, route_name='apex_login',
                    renderer=render_template, permission=NO_PERMISSION_REQUIRED)

    config.add_route('apex_logout', '/logout')
    config.add_view(logout, route_name='apex_logout',
                    renderer=render_template, permission=NO_PERMISSION_REQUIRED)

    config.add_route('apex_register', '/register')
    config.add_view(register, route_name='apex_register',
                    renderer=render_template, permission=NO_PERMISSION_REQUIRED)

    config.add_route('apex_password', '/password')
    config.add_view(change_password, route_name='apex_password',
                    renderer=render_template, permission='authenticated')

    config.add_route('apex_forgot', '/forgot')
    config.add_view(forgot_password, route_name='apex_forgot',
                    renderer=render_template, permission=NO_PERMISSION_REQUIRED)

    config.add_route('apex_reset', '/reset/:user_id/:hmac')
    config.add_view(reset_password, route_name='apex_reset',
                    renderer=render_template, permission=NO_PERMISSION_REQUIRED)

    config.add_route('apex_activate', '/activate/:user_id/:hmac')
    config.add_view(activate, route_name='apex_activate',
                    renderer=render_template, permission=NO_PERMISSION_REQUIRED)

    config.add_route('apex_callback', '/apex_callback')
    config.add_view(apex_callback, route_name='apex_callback', permission=NO_PERMISSION_REQUIRED)

    config.add_route('apex_openid_required', '/openid_required')
    config.add_view(openid_required, route_name= \
                    'apex_openid_required', \
                    renderer=render_template, permission=NO_PERMISSION_REQUIRED)

    if settings.has_key('apex.auth_profile'):
        use_edit = asbool(settings.get('apex.use_apex_edit', False))
        if use_edit:
            config.add_route('apex_edit', '/edit')
            config.add_view(edit, route_name='apex_edit', \
                            renderer=render_template, \
                            permission='authenticated')
<|endoftext|>


<commit_before>import logging
from lib.sforce.base import SforceBaseClient
from suds import WebFault
from lib.sforce.partner import SforcePartnerClient
from lib.sforce.metadata import SforceMetadataClient
from lib.sforce.apex import SforceApexClient
from lib.sforce.tooling import SforceToolingClient
from optparse import OptionParser

logging.basicConfig(format='[=] %(asctime)s | %(levelname)s | %(message)s [%(module)s:%(lineno)d]  ',datefmt='%m/%d/%Y %I:%M:%S %p',level=logging.DEBUG)
logging.info('Loading libs')
import lib.mm_util as mm_util
import time

import os

# Adds an option to command line to clean up all transactions and mappings on start
# for dev purposes only.
parser = OptionParser()

parser.add_option("-u", "--user", dest="user", type="string", help="Salesforce username")
parser.add_option("-p", "--password", dest="password", type="string", help="Salesforce password")
parser.add_option("-t", "--token", dest="token", type="string", help="Salesforce token")
parser.add_option("-s", "--apex-script", dest="apexscriptfilename", type="string", help="Apex code to execute")

(options, args) = parser.parse_args()

missing_args = False
error_log = '\n [!] Errors found \n\n'
if options.user == None:
      missing_args = True
      error_log = error_log + "~ Salesforce username is required \n"
if options.password == None:
      missing_args = True
      error_log = error_log + "~ Salesforce password is required \n"
if options.apexscriptfilename == None:
      missing_args = True
      error_log = error_log + "~ Apex script filename is required \n"

if missing_args:
      print error_log
else:
      logging.info('Loading partner WSDL')
      wsdl_location = os.path.join(mm_util.WSDL_PATH, 'partner.xml')
      client = SforcePartnerClient(
                  wsdl_location,
                  apiVersion=None,
                  environment='production',
                  sid=None,
                  metadata_server_url=None,
                  server_url=None)

      try:
            # login using partner wsdl
            logging.info('Authenticating')
            client.login(options.user,options.password,options.token)

            # use token with apex wsdl
            apex_wsdl_location = os.path.join(mm_util.WSDL_PATH, 'apex.xml')
            apex_client = SforceApexClient(
                        apex_wsdl_location,
                        apiVersion=mm_util.SFDC_API_VERSION,
                        environment='production',
                        sid=client.getSessionId(),
                        metadata_server_url=client.getMetadaServerUrl(),
                        server_url=mm_util.get_sfdc_endpoint_by_type('enterprise'))

            # open script file
            logging.info('Loading scripts')
            f = open(options.apexscriptfilename, "r")
            apex_code = f.read()

            # Execute code
            logging.info('Running apex code')
            t0 = time.clock()
            apex_execution = apex_client.executeAnonymous({"body":apex_code})
            logging.info('Finished execution')
            logging.info("Code executed in "+str(time.clock() - t0)+ " seconds.")
            if apex_execution.success:
                  print "~ Script executed successfully"
            else:
                  print "[!] Errors found:"
                  if apex_execution.exceptionMessage:
                        print apex_execution.exceptionMessage
                  if apex_execution.compileProblem:
                        print 'Compilation error: '+apex_execution.compileProblem
                  print 'Line: '+str(apex_execution.line)

      except Exception, e:
            logging.error('Exception found:'+str(e.message))

<commit_msg>removed token as required parameter<commit_after>import logging
from lib.sforce.base import SforceBaseClient
from suds import WebFault
from lib.sforce.partner import SforcePartnerClient
from lib.sforce.metadata import SforceMetadataClient
from lib.sforce.apex import SforceApexClient
from lib.sforce.tooling import SforceToolingClient
from optparse import OptionParser

logging.basicConfig(format='[=] %(asctime)s | %(levelname)s | %(message)s [%(module)s:%(lineno)d]  ',datefmt='%m/%d/%Y %I:%M:%S %p',level=logging.DEBUG)
logging.info('Loading libs')
import lib.mm_util as mm_util
import time

import os

# Adds an option to command line to clean up all transactions and mappings on start
# for dev purposes only.
parser = OptionParser()

parser.add_option("-u", "--user", dest="user", type="string", help="Salesforce username")
parser.add_option("-p", "--password", dest="password", type="string", help="Salesforce password")
parser.add_option("-t", "--token", dest="token", type="string", help="Salesforce token")
parser.add_option("-s", "--apex-script", dest="apexscriptfilename", type="string", help="Apex code to execute")

(options, args) = parser.parse_args()

missing_args = False
error_log = '\n [!] Errors found \n\n'
if options.user == None:
      missing_args = True
      error_log = error_log + "~ Salesforce username is required \n"
if options.password == None:
      missing_args = True
      error_log = error_log + "~ Salesforce password is required \n"
if options.apexscriptfilename == None:
      missing_args = True
      error_log = error_log + "~ Apex script filename is required \n"

if missing_args:
      print error_log
else:
      logging.info('Loading partner WSDL')
      wsdl_location = os.path.join(mm_util.WSDL_PATH, 'partner.xml')
      client = SforcePartnerClient(
                  wsdl_location,
                  apiVersion=None,
                  environment='production',
                  sid=None,
                  metadata_server_url=None,
                  server_url=None)

      try:
            # login using partner wsdl
            logging.info('Authenticating')
            # sometimes password and token are provided together.
            # token parameter is not required.
            token_safe = ''
            if options.token:
                  token_safe = options.token
            client.login(options.user,options.password,token_safe)

            # use token with apex wsdl
            apex_wsdl_location = os.path.join(mm_util.WSDL_PATH, 'apex.xml')
            apex_client = SforceApexClient(
                        apex_wsdl_location,
                        apiVersion=mm_util.SFDC_API_VERSION,
                        environment='production',
                        sid=client.getSessionId(),
                        metadata_server_url=client.getMetadaServerUrl(),
                        server_url=mm_util.get_sfdc_endpoint_by_type('enterprise'))

            # open script file
            logging.info('Loading scripts')
            f = open(options.apexscriptfilename, "r")
            apex_code = f.read()

            # Execute code
            logging.info('Running apex code')
            t0 = time.clock()
            apex_execution = apex_client.executeAnonymous({"body":apex_code})
            logging.info('Finished execution')
            logging.info("Code executed in "+str(time.clock() - t0)+ " seconds.")
            if apex_execution.success:
                  print "~ Script executed successfully"
            else:
                  print "[!] Errors found:"
                  if apex_execution.exceptionMessage:
                        print apex_execution.exceptionMessage
                  if apex_execution.compileProblem:
                        print 'Compilation error: '+apex_execution.compileProblem
                  print 'Line: '+str(apex_execution.line)

      except Exception, e:
            logging.error('Exception found:'+str(e.message))

<|endoftext|>


<commit_before>
import os, imp, getpass, sys, traceback, re

from twisted.python.filepath import FilePath
from twisted.internet.task import react
from twisted.internet import reactor
from twisted.internet.endpoints import UNIXClientEndpoint
from twisted.conch.ssh.keys import EncryptedKeyError, Key
from twisted.conch.client.knownhosts import KnownHostsFile
from twisted.python import log as logger

import click

from structlog import PrintLogger
log = PrintLogger()

from plait.app.console import ConsoleApp
from plait.app.terminal import TerminalApp
from plait.runner import PlaitRunner
from plait.task import NoSuchTaskError, task
from plait.errors import *
from plait.utils import parse_task_calls, Bag

def findPlaitfile(path=os.getcwd()):
    files = os.listdir(path)
    parent = os.path.dirname(path)
    if 'plaitfile.py' in files:
        fullpath = os.path.join(path, 'plaitfile.py')
        return os.path.abspath(fullpath)
    elif parent != path:
        return findPlaitfile(parent)
    raise StartupError("Couldn't find plaitfile.py")

def importPlaitfile(filename):
    try:
        err = None
        return imp.load_source('plaitfile', filename)
    except Exception as e:
        e = StartupError("Couldn't import plaitfile")
        e.tb = traceback.format_exc()
        raise e

def gatherTasks(tasks, pf_module):
    parsed_tasks = parse_task_calls(tasks)
    for task_name, args, kwargs in parsed_tasks:
        task_func = getattr(pf_module, task_name, None)
        if not task_func:
            error = "Task `{}` does not exist in pf_module: {}"
            error = error.format(task_name, pf_module.__file__)
            raise NoSuchTaskError(error)
        task_func = task(task_func)
        yield task_name, task_func, args, kwargs

def getTasks(tasks, plaitfile, **kwargs):
    if not tasks:
        raise StartupError("Must specify at least one task to execute.")

    pf_module = importPlaitfile(plaitfile or findPlaitfile())
    return list(gatherTasks(tasks, pf_module))

def setupLogging(logging, **kwargs):
    if logging:
        logger.startLogging(sys.stdout, setStdout=1)

def readKey(path):
    try:
        return Key.fromFile(path)
    except EncryptedKeyError:
        passphrase = getpass.getpass("%r keyphrase: " % (path,))
        return Key.fromFile(path, passphrase=passphrase)

def getKeys(identity, **kwargs):
    key_path = os.path.expanduser(identity)
    if os.path.exists(key_path):
        return [readKey(key_path)]

def getKnownHosts(knownhosts, **kwargs):
    known_hosts_path = FilePath(os.path.expanduser(knownhosts))
    if known_hosts_path.exists():
        return KnownHostsFile.fromPath(known_hosts_path)

def getAgentEndpoint(agent, **kwargs):
    if "SSH_AUTH_SOCK" in os.environ and agent:
        auth_socket = os.environ["SSH_AUTH_SOCK"]
        return UNIXClientEndpoint(reactor, auth_socket)

def getHosts(host, hostfile, **kwargs):
    hosts = list(host)
    if hostfile:
        with open(hostfile, 'r') as fobj:
            hosts += tuple(l.strip() for l in fobj.readlines())
    if not sys.stdin.isatty():
        hosts += tuple(l.strip() for l in sys.stdin.readlines())
    if not len(hosts):
        raise StartupError("Must specify at least one host.")
    return hosts

def getErrorFilter(errors, hide_errors, **kwargs):
    if errors and hide_errors:
        msg = "`-e` and `-E` cannot be used simultaneously."
        raise StartupError(msg)
    elif errors:
        # only show errors
        return True
    elif hide_errors:
        # don't show errors
        return False
    else:
        # show all sessions
        return None

def getGrepFilter(grep, hide_grep, **kwargs):
    if grep and hide_grep:
        msg = "`-g` and `-G` cannot be used simultaneously."
        raise StartupError(msg)
    elif grep:
        return lambda x: re.search(grep, x)
    elif hide_grep:
        return lambda x: re.search(hide_grep, x) is None
    else:
        return lambda x: re.search(".*", x)

def getConnectSettings(scale, retries, timeout, **kwargs):
    return Bag(scale=scale,
               retries=retries,
               timeout=timeout,
               keys = getKeys(**kwargs),
               known_hosts = getKnownHosts(**kwargs),
               agent_endpoint = getAgentEndpoint(**kwargs))

@click.command()
@click.argument('tasks', nargs=-1)
@click.option('--interactive', '-I',
              is_flag=True,
              help="Display results graphically")
@click.option('--plaitfile', '-p',
              default=None, metavar='',
              help="Read tasks from specified file")
@click.option('--host', '-h',
              multiple=True, metavar='*',
              help="[$USER@]hostname[:22]")
@click.option('--hostfile', '-H',
              default=False, metavar='',
              help="Read hosts from a line delimited file")
@click.option('--scale', '-s',
              default=0, metavar='',
              help="Number of hosts to execute in parallel")
@click.option('--errors', '-e',
              is_flag=True, metavar='',
              help="Only show sessions with an error")
@click.option('--hide-errors', '-E',
              is_flag=True, metavar='',
              help="Hide sessions with an error")
@click.option('--grep', '-g',
              default=None, metavar='',
              help="Only display sessions matching a pattern")
@click.option('--hide-grep', '-G',
              default=None, metavar='',
              help="Hide sessions matching a pattern")
@click.option('--identity', '-i',
              default="~/.ssh/id_rsa", metavar="*",
              help="Public key to use")
@click.option('--agent', '-a',
              is_flag=True,
              help="Whether to use system ssh-agent for auth")
@click.option('--knownhosts', '-k',
              default="~/.ssh/known_hosts", metavar='',
              help="File with authorized hosts")
@click.option('--retries', '-r',
              default=1, metavar='',
              help="Times to retry SSH connection")
@click.option('--timeout', '-t',
              default=10, metavar='',
              help="Seconds to wait for SSH")
@click.option('--logging', '-l',
              is_flag=True,
              help="Show twisted logging")
def run(tasks, interactive, **kwargs):
    """
    * can be supplied multiple times
    """
    setupLogging(**kwargs)

    tasks = getTasks(tasks, **kwargs)
    hosts = getHosts(**kwargs)
    connect_settings = getConnectSettings(**kwargs)
    runner = PlaitRunner(hosts, tasks, connect_settings)

    errorFilter = getErrorFilter(**kwargs)
    grepFilter = getGrepFilter(**kwargs)

    if interactive:
        console = ConsoleApp(title="plait 1.0")
    else:
        console = TerminalApp(errorFilter, grepFilter)

    console.run(runner)
def main():
    try:
        run()
    except StartupError as e:
        log.error(" * " + e.message)
        if hasattr(e, 'tb'):
            log.error(e.tb)
<commit_msg>Make the grepping logic a bit more explicit<commit_after>
import os, imp, getpass, sys, traceback, re

from twisted.python.filepath import FilePath
from twisted.internet.task import react
from twisted.internet import reactor
from twisted.internet.endpoints import UNIXClientEndpoint
from twisted.conch.ssh.keys import EncryptedKeyError, Key
from twisted.conch.client.knownhosts import KnownHostsFile
from twisted.python import log as logger

import click

from structlog import PrintLogger
log = PrintLogger()

from plait.app.console import ConsoleApp
from plait.app.terminal import TerminalApp
from plait.runner import PlaitRunner
from plait.task import NoSuchTaskError, task
from plait.errors import *
from plait.utils import parse_task_calls, Bag

def findPlaitfile(path=os.getcwd()):
    files = os.listdir(path)
    parent = os.path.dirname(path)
    if 'plaitfile.py' in files:
        fullpath = os.path.join(path, 'plaitfile.py')
        return os.path.abspath(fullpath)
    elif parent != path:
        return findPlaitfile(parent)
    raise StartupError("Couldn't find plaitfile.py")

def importPlaitfile(filename):
    try:
        err = None
        return imp.load_source('plaitfile', filename)
    except Exception as e:
        e = StartupError("Couldn't import plaitfile")
        e.tb = traceback.format_exc()
        raise e

def gatherTasks(tasks, pf_module):
    parsed_tasks = parse_task_calls(tasks)
    for task_name, args, kwargs in parsed_tasks:
        task_func = getattr(pf_module, task_name, None)
        if not task_func:
            error = "Task `{}` does not exist in pf_module: {}"
            error = error.format(task_name, pf_module.__file__)
            raise NoSuchTaskError(error)
        task_func = task(task_func)
        yield task_name, task_func, args, kwargs

def getTasks(tasks, plaitfile, **kwargs):
    if not tasks:
        raise StartupError("Must specify at least one task to execute.")

    pf_module = importPlaitfile(plaitfile or findPlaitfile())
    return list(gatherTasks(tasks, pf_module))

def setupLogging(logging, **kwargs):
    if logging:
        logger.startLogging(sys.stdout, setStdout=1)

def readKey(path):
    try:
        return Key.fromFile(path)
    except EncryptedKeyError:
        passphrase = getpass.getpass("%r keyphrase: " % (path,))
        return Key.fromFile(path, passphrase=passphrase)

def getKeys(identity, **kwargs):
    key_path = os.path.expanduser(identity)
    if os.path.exists(key_path):
        return [readKey(key_path)]

def getKnownHosts(knownhosts, **kwargs):
    known_hosts_path = FilePath(os.path.expanduser(knownhosts))
    if known_hosts_path.exists():
        return KnownHostsFile.fromPath(known_hosts_path)

def getAgentEndpoint(agent, **kwargs):
    if "SSH_AUTH_SOCK" in os.environ and agent:
        auth_socket = os.environ["SSH_AUTH_SOCK"]
        return UNIXClientEndpoint(reactor, auth_socket)

def getHosts(host, hostfile, **kwargs):
    hosts = list(host)
    if hostfile:
        with open(hostfile, 'r') as fobj:
            hosts += tuple(l.strip() for l in fobj.readlines())
    if not sys.stdin.isatty():
        hosts += tuple(l.strip() for l in sys.stdin.readlines())
    if not len(hosts):
        raise StartupError("Must specify at least one host.")
    return hosts

def getErrorFilter(errors, hide_errors, **kwargs):
    if errors and hide_errors:
        msg = "`-e` and `-E` cannot be used simultaneously."
        raise StartupError(msg)
    elif errors:
        # only show errors
        return True
    elif hide_errors:
        # don't show errors
        return False
    else:
        # show all sessions
        return None

def getGrepFilter(grep, hide_grep, **kwargs):
    if grep and hide_grep:
        msg = "`-g` and `-G` cannot be used simultaneously."
        raise StartupError(msg)
    elif grep:
        return lambda x: (re.search(grep, x) is not None)
    elif hide_grep:
        return lambda x: (re.search(hide_grep, x) is None)
    else:
        return lambda x: (re.search(".*", x) is not None)

def getConnectSettings(scale, retries, timeout, **kwargs):
    return Bag(scale=scale,
               retries=retries,
               timeout=timeout,
               keys = getKeys(**kwargs),
               known_hosts = getKnownHosts(**kwargs),
               agent_endpoint = getAgentEndpoint(**kwargs))

@click.command()
@click.argument('tasks', nargs=-1)
@click.option('--interactive', '-I',
              is_flag=True,
              help="Display results graphically")
@click.option('--plaitfile', '-p',
              default=None, metavar='',
              help="Read tasks from specified file")
@click.option('--host', '-h',
              multiple=True, metavar='*',
              help="[$USER@]hostname[:22]")
@click.option('--hostfile', '-H',
              default=False, metavar='',
              help="Read hosts from a line delimited file")
@click.option('--scale', '-s',
              default=0, metavar='',
              help="Number of hosts to execute in parallel")
@click.option('--errors', '-e',
              is_flag=True, metavar='',
              help="Only show sessions with an error")
@click.option('--hide-errors', '-E',
              is_flag=True, metavar='',
              help="Hide sessions with an error")
@click.option('--grep', '-g',
              default=None, metavar='',
              help="Only display sessions matching a pattern")
@click.option('--hide-grep', '-G',
              default=None, metavar='',
              help="Hide sessions matching a pattern")
@click.option('--identity', '-i',
              default="~/.ssh/id_rsa", metavar="*",
              help="Public key to use")
@click.option('--agent', '-a',
              is_flag=True,
              help="Whether to use system ssh-agent for auth")
@click.option('--knownhosts', '-k',
              default="~/.ssh/known_hosts", metavar='',
              help="File with authorized hosts")
@click.option('--retries', '-r',
              default=1, metavar='',
              help="Times to retry SSH connection")
@click.option('--timeout', '-t',
              default=10, metavar='',
              help="Seconds to wait for SSH")
@click.option('--logging', '-l',
              is_flag=True,
              help="Show twisted logging")
def run(tasks, interactive, **kwargs):
    """
    * can be supplied multiple times
    """
    setupLogging(**kwargs)

    tasks = getTasks(tasks, **kwargs)
    hosts = getHosts(**kwargs)
    connect_settings = getConnectSettings(**kwargs)
    runner = PlaitRunner(hosts, tasks, connect_settings)

    errorFilter = getErrorFilter(**kwargs)
    grepFilter = getGrepFilter(**kwargs)

    if interactive:
        console = ConsoleApp(title="plait 1.0")
    else:
        console = TerminalApp(errorFilter, grepFilter)

    console.run(runner)
def main():
    try:
        run()
    except StartupError as e:
        log.error(" * " + e.message)
        if hasattr(e, 'tb'):
            log.error(e.tb)
<|endoftext|>


<commit_before>import argparse
import os
import platform
import shlex
import subprocess
import sys
from subprocess import CalledProcessError, list2cmdline

from . import core, dockercore, state
from .core import BuildError
from .dockercore import DockerBuildError, docker_get_image_shell
from .eval import ActionNotFoundError, ArgError, EvalError, eval, layer
from .utils import (disable_friendly_exception, friendly_exception, hashstr,
                    rand)

HELP = 'my help'
PROG = 'plash'

NO_TERM_BUILD_ERROR = """plash error: Refusing to build when not connected to a tty(-like) device.
Set the env PLASH_BUILD_SILENT to enable building without output is such cases.
Or invoke this call with --build-only from a terminal to build and then run again.
The argv of this program was: {}""".format(sys.argv)

# shortcuts are specific to the stdlib
SHORTCUTS = [
    # shortcut, lsp, nargs
    ('-x', ['run'], '+'),
    ('-a', ['apt'], '+'),
    ('-y', ['yum'], '+'),
    ('-p', ['pip'], '+'),
    ('-b', ['apt', 'ubuntu-server'], 0),
    ('-o', ['os'], 1),
    ('-r', ['include', '~/.plashrc'], 0),
    ('-U', ['os', 'ubuntu'], 0),
    ('-F', ['os', 'fedora'], 0),
    ('-D', ['os', 'debian'], 0),
 ]

def add_shortcuts_to_parser(parser):
    group = parser.add_argument_group('shortcuts', 'shortcuts')
    for shortcut, lsp, nargs in SHORTCUTS:
        group.add_argument(
            shortcut,
            action=create_collect_lsp_action(lsp),
            nargs=nargs)

def create_collect_lsp_action(lsp_begin):
    class CollectAction(argparse.Action):
        def __call__(self, parser, namespace, values, option_string=None):
            if not 'lsp' in namespace:
                setattr(namespace, 'lsp', [])
            previous = namespace.lsp
            previous.append(lsp_begin + list(values))
            setattr(namespace, 'lsp', previous)
    return CollectAction

def get_argument_parser():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawTextHelpFormatter,
        description='Run programm from any Linux',
        prog=PROG,
        epilog=HELP)

    parser.add_argument("subcommand", nargs='?')

    parser.add_argument("--build-silent", action='store_true')
    parser.add_argument("--build-verbose", "--build-loud", action='store_true', dest='verbose')
    parser.add_argument("--build-only", action='store_true')
    parser.add_argument("--build-again", "--rebuild", "--again", action='store_true')

    parser.add_argument("--no-stdlib", action='store_true')
    parser.add_argument("--traceback", action='store_true')
    parser.add_argument("--debug-lsp", action='store_true')

    parser.add_argument("--ignore-no-tty", action='store_false')

    parser.add_argument("--docker", "-d", action='store_true')

    parser.add_argument("--docker-save-image")

    add_shortcuts_to_parser(parser)

    return parser

def main():
    argv = sys.argv[1:]

    try:
        delimiter = argv.index('--')
        command = argv[delimiter+1:]
        argv = argv[:delimiter]
        # assert False, (args, command)
    except ValueError:
        command = None

    ap = get_argument_parser()
    _, unused_args = ap.parse_known_args(argv)
    # lsp = unused_args_to_lsp(unused_args)
    for arg in set(unused_args):
        if arg == '--':
            break
        if arg.startswith('--'):
            ap.add_argument(
                arg,
                action=create_collect_lsp_action([arg[2:]]),
                nargs='*')
    args = ap.parse_args(argv)

    if args.subcommand:
        if argv[0].startswith('-'):
            ap.error('Subcommand must always be the first argument')
        else:

            this_file_folder = os.path.abspath(os.path.dirname(__file__))
            subcommand_executable = os.path.join(this_file_folder, 'subcommands', args.subcommand)
            os.execvpe(
                subcommand_executable,
                [subcommand_executable] + argv[1:],
                dict(os.environ, PLASH_DATA=core.BASE_DIR))

    build_silent = args.build_silent or os.environ.get('PLASH_BUILD_SILENT')
    lsp = getattr(args, 'lsp', [])

    if args.traceback:
        disable_friendly_exception()
    if not args.no_stdlib:
        init = [['import', 'plash.stdlib']]
    else:
        init = []

    if args.debug_lsp:
        print(init + lsp)
        sys.exit(0)
    with friendly_exception([ActionNotFoundError, ArgError, EvalError]):
        state.reset()
        script = eval(init + lsp)

    os_image = state.get_os()
    if not os_image:
        ap.error('Specify an image')
    layers = (script + '\n').split(layer() + '\n')
    layers = [l for l in layers if l]
    if os_image == 'print':
        print(script)
        sys.exit(0)

    if os_image.startswith('build://'):
        build = os_image[len('build://'):]
        tmp_image = rand() # fixme cleanup this image later
        p = subprocess.Popen(['docker', 'build', build, '-t', tmp_image])
        exit = p.wait()
        assert exit == 0
        image = subprocess.check_output(
            ['docker', 'images', '--quiet', tmp_image])
        image = image.decode().rstrip('\n')
        subprocess.check_output(['docker', 'rmi', tmp_image])
    else:
        image = os_image

    # b = LayeredDockerBuildable.create(image, layers)
    # with friendly_exception([BuildError, CalledProcessError]):
    #     if args.build_again or not b.image_ready():
    #         if not sys.stdout.isatty() and not build_silent and not args.ignore_no_tty:
    #             sys.stderr.write(NO_TERM_BUILD_ERROR)
    #             print()
    #             sys.exit(127)

        plash_env = '{}-{}'.format( # some
            os_image,
            hashstr('\n'.join(layers).encode())[:4])

        if args.docker or platform.system() in ('Darwin', 'Windows'):
            execute = dockercore.execute
            execute_extra_kws = {'docker_save_image': args.docker_save_image}
        else:
            execute = core.execute
            execute_extra_kws = {}

        # bcmd = state.get_base_command() or ''
        # command = (command or [docker_get_image_shell(image)]) if not bcmd else shlex.split(bcmd) + (command or [])

        execute(image,
                layers,
                command,
                quiet_flag=args.build_silent,
                rebuild_flag=args.build_again,
                verbose_flag=args.verbose,
                build_only=not command,
                skip_if_exists=not args.build_again,
                extra_mounts=state.pop_mountpoints(),
                **execute_extra_kws)

        # # something that could be used in the shell prompt
        # docker_run(
        #     b.get_image_name(),
        #     command,
        #     extra_envs={'PLASH_ENV': plash_env},
        #     )

        #     print('*** plash: building...')
        #     b.build(
        #         shell=docker_get_image_shell(image),
        #         quiet=build_silent,
        #         verbose=args.verbose,
        #         extra_mounts=state.pop_mountpoints(),
        #         skip_if_exists=not args.build_again)
        #     print('*** plash: done')
        #     print()
<commit_msg>Fixes in subcommand handling<commit_after>import argparse
import os
import platform
import shlex
import subprocess
import sys
from subprocess import CalledProcessError, list2cmdline

from . import core, dockercore, state
from .core import BuildError
from .dockercore import DockerBuildError, docker_get_image_shell
from .eval import ActionNotFoundError, ArgError, EvalError, eval, layer
from .utils import (disable_friendly_exception, friendly_exception, hashstr,
                    rand)

HELP = 'my help'
PROG = 'plash'

NO_TERM_BUILD_ERROR = """plash error: Refusing to build when not connected to a tty(-like) device.
Set the env PLASH_BUILD_SILENT to enable building without output is such cases.
Or invoke this call with --build-only from a terminal to build and then run again.
The argv of this program was: {}""".format(sys.argv)

# shortcuts are specific to the stdlib
SHORTCUTS = [
    # shortcut, lsp, nargs
    ('-x', ['run'], '+'),
    ('-a', ['apt'], '+'),
    ('-y', ['yum'], '+'),
    ('-p', ['pip'], '+'),
    ('-b', ['apt', 'ubuntu-server'], 0),
    ('-o', ['os'], 1),
    ('-r', ['include', '~/.plashrc'], 0),
    ('-U', ['os', 'ubuntu'], 0),
    ('-F', ['os', 'fedora'], 0),
    ('-D', ['os', 'debian'], 0),
 ]

def add_shortcuts_to_parser(parser):
    group = parser.add_argument_group('shortcuts', 'shortcuts')
    for shortcut, lsp, nargs in SHORTCUTS:
        group.add_argument(
            shortcut,
            action=create_collect_lsp_action(lsp),
            nargs=nargs)

def create_collect_lsp_action(lsp_begin):
    class CollectAction(argparse.Action):
        def __call__(self, parser, namespace, values, option_string=None):
            if not 'lsp' in namespace:
                setattr(namespace, 'lsp', [])
            previous = namespace.lsp
            previous.append(lsp_begin + list(values))
            setattr(namespace, 'lsp', previous)
    return CollectAction

def get_argument_parser():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawTextHelpFormatter,
        description='Run programm from any Linux',
        prog=PROG,
        epilog=HELP)

    # parser.add_argument("subcommand", nargs='?')
    parser.add_argument("--build-silent", action='store_true')
    parser.add_argument("--build-verbose", "--build-loud", action='store_true', dest='verbose')
    parser.add_argument("--build-only", action='store_true')
    parser.add_argument("--build-again", "--rebuild", "--again", action='store_true')

    parser.add_argument("--no-stdlib", action='store_true')
    parser.add_argument("--traceback", action='store_true')
    parser.add_argument("--debug-lsp", action='store_true')

    parser.add_argument("--ignore-no-tty", action='store_false')

    parser.add_argument("--docker", "-d", action='store_true')

    parser.add_argument("--docker-save-image")

    add_shortcuts_to_parser(parser)

    return parser

def main():
    argv = sys.argv[1:]

    if not argv[0].startswith('-'): # suppose its a subcommand
        subcommand = argv[0]
        subcommands_dir = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            'subcommands')
        all_subcommands = os.listdir(subcommands_dir)
        if not subcommand in all_subcommands:
            with friendly_exception([ArgError]):
                raise ArgError('no subcommand {}, available: {}'.format(repr(subcommand), ' '.join(all_subcommands)))

        subcommand_executable = os.path.join(subcommands_dir, subcommand)
        os.execvpe(
            subcommand_executable,
            [subcommand_executable] + argv[1:],
            dict(os.environ, PLASH_DATA=core.BASE_DIR))

    try:
        delimiter = argv.index('--')
        command = argv[delimiter+1:]
        argv = argv[:delimiter]
        # assert False, (args, command)
    except ValueError:
        command = None

    ap = get_argument_parser()
    _, unused_args = ap.parse_known_args(argv)
    # lsp = unused_args_to_lsp(unused_args)
    for arg in set(unused_args):
        if arg == '--':
            break
        if arg.startswith('--'):
            ap.add_argument(
                arg,
                action=create_collect_lsp_action([arg[2:]]),
                nargs='*')

    args = ap.parse_args(argv)

    build_silent = args.build_silent or os.environ.get('PLASH_BUILD_SILENT')
    lsp = getattr(args, 'lsp', [])

    if args.traceback:
        disable_friendly_exception()
    if not args.no_stdlib:
        init = [['import', 'plash.stdlib']]
    else:
        init = []

    if args.debug_lsp:
        print(init + lsp)
        sys.exit(0)
    with friendly_exception([ActionNotFoundError, ArgError, EvalError]):
        state.reset()
        script = eval(init + lsp)

    os_image = state.get_os()
    if not os_image:
        ap.error('Specify an image')
    layers = (script + '\n').split(layer() + '\n')
    layers = [l for l in layers if l]
    if os_image == 'print':
        print(script)
        sys.exit(0)

    if os_image.startswith('build://'):
        build = os_image[len('build://'):]
        tmp_image = rand() # fixme cleanup this image later
        p = subprocess.Popen(['docker', 'build', build, '-t', tmp_image])
        exit = p.wait()
        assert exit == 0
        image = subprocess.check_output(
            ['docker', 'images', '--quiet', tmp_image])
        image = image.decode().rstrip('\n')
        subprocess.check_output(['docker', 'rmi', tmp_image])
    else:
        image = os_image

    # b = LayeredDockerBuildable.create(image, layers)
    # with friendly_exception([BuildError, CalledProcessError]):
    #     if args.build_again or not b.image_ready():
    #         if not sys.stdout.isatty() and not build_silent and not args.ignore_no_tty:
    #             sys.stderr.write(NO_TERM_BUILD_ERROR)
    #             print()
    #             sys.exit(127)

        plash_env = '{}-{}'.format( # some
            os_image,
            hashstr('\n'.join(layers).encode())[:4])

        if args.docker or platform.system() in ('Darwin', 'Windows'):
            execute = dockercore.execute
            execute_extra_kws = {'docker_save_image': args.docker_save_image}
        else:
            execute = core.execute
            execute_extra_kws = {}

        # bcmd = state.get_base_command() or ''
        # command = (command or [docker_get_image_shell(image)]) if not bcmd else shlex.split(bcmd) + (command or [])

        execute(image,
                layers,
                command,
                quiet_flag=args.build_silent,
                rebuild_flag=args.build_again,
                verbose_flag=args.verbose,
                build_only=not command,
                skip_if_exists=not args.build_again,
                extra_mounts=state.pop_mountpoints(),
                **execute_extra_kws)

        # # something that could be used in the shell prompt
        # docker_run(
        #     b.get_image_name(),
        #     command,
        #     extra_envs={'PLASH_ENV': plash_env},
        #     )

        #     print('*** plash: building...')
        #     b.build(
        #         shell=docker_get_image_shell(image),
        #         quiet=build_silent,
        #         verbose=args.verbose,
        #         extra_mounts=state.pop_mountpoints(),
        #         skip_if_exists=not args.build_again)
        #     print('*** plash: done')
        #     print()
<|endoftext|>


<commit_before>""" Module for getting competition statistics"""

import api

from api.common import cache, APIException

_get_problem_names = lambda problems: [problem['name'] for problem in problems]

@api.cache.memoize()
def get_score(tid=None, uid=None):
    """
    Get the score for a user or team.

    Args:
        tid: The team id
        uid: The user id
    Returns:
        The users's or team's score
    """

    score = sum([problem['score'] for problem in api.problem.get_solved_problems(tid=tid, uid=uid)])

    return score

def get_group_scores(gid=None, name=None):
    """
    Get the group scores.

    Args:
        gid: The group id
        name: The group name
    Returns:
        A dictionary of tid:score mappings
    """

    members = [api.team.get_team(tid=tid) for tid in api.group.get_group(gid, name)['members']]

    result = []
    for team in members:
        result.append({
            "name": team['team_name'],
            "score": get_score(tid=team['tid'])
        })

    return sorted(result, key=lambda entry: entry['score'], reverse=True)

def get_group_average_score(gid=None, name=None):
    """
    Get the average score of teams in a group.

    Args:
        gid: The group id
        name: The group name
    Returns:
        The total score of the group
    """

    group_scores = get_group_scores(gid=gid, name=name)
    total_score = sum([entry['score'] for entry in group_scores])
    return int(total_score / len(group_scores)) if len(group_scores) > 0 else 0

@api.cache.memoize()
def get_all_team_scores():
    """
    Gets the score for every team in the database.

    Returns:
        A list of dictionaries with name and score
    """

    teams = api.team.get_all_teams()

    result = []
    for team in teams:
        result.append({
            "name": team['team_name'],
            "score": get_score(tid=team['tid'])
        })

    return sorted(result, key=lambda entry: entry['score'], reverse=True)

def get_all_user_scores():
    """
    Gets the score for every user in the database.

    Returns:
        A list of dictionaries with name and score
    """

    users = api.user.get_all_users()

    result = []
    for user in users:
        result.append({
            "name": user['username'],
            "score": get_score(uid=user['uid'])
        })

    return sorted(result, key=lambda entry: entry['score'], reverse=True)

@api.cache.memoize(timeout=120, fast=True)
def get_problems_by_category():
    """
    Gets the list of all problems divided into categories

    Returns:
        A dictionary of category:[problem list]
    """

    result = {cat: _get_problem_names(api.problem.get_all_problems(category=cat))
                            for cat in api.problem.get_all_categories()}

    return result

def get_team_member_stats(tid):
    members = api.team.get_team_members(tid=tid)

    return {member['username']: _get_problem_names(api.problem.get_solved_problems(uid=member['uid'])) for member in members}

#@api.cache.memoize()
def get_score_over_time(uid=None, tid=None, category=None):
    """
    Finds the score and time after each correct submission of a team or user.
    NOTE: this is slower than get_score. Do not use this for getting current score.

    Args:
        uid: the uid of the user
        tid: the tid of the user
        category: category filter
    Returns:
        A list of dictionaries containing score and time
    """

    correct_submissions = api.problem.get_submissions(uid=uid, tid=tid, category=category, correctness=True)

    result = []
    score = 0

    for submission in sorted(correct_submissions, key=lambda sub: sub["timestamp"]):
        score += api.problem.get_problem(pid=submission["pid"])["score"]
        result.append({
            "score": score,
            "time": int(submission["timestamp"].timestamp())
        })

    return result
<commit_msg>Implemented top teams score progression<commit_after>""" Module for getting competition statistics"""

import api

from api.common import cache, APIException

_get_problem_names = lambda problems: [problem['name'] for problem in problems]

top_teams = 10

@api.cache.memoize()
def get_score(tid=None, uid=None):
    """
    Get the score for a user or team.

    Args:
        tid: The team id
        uid: The user id
    Returns:
        The users's or team's score
    """

    score = sum([problem['score'] for problem in api.problem.get_solved_problems(tid=tid, uid=uid)])

    return score

def get_group_scores(gid=None, name=None):
    """
    Get the group scores.

    Args:
        gid: The group id
        name: The group name
    Returns:
        A dictionary of tid:score mappings
    """

    members = [api.team.get_team(tid=tid) for tid in api.group.get_group(gid, name)['members']]

    result = []
    for team in members:
        result.append({
            "name": team['team_name'],
            "score": get_score(tid=team['tid'])
        })

    return sorted(result, key=lambda entry: entry['score'], reverse=True)

def get_group_average_score(gid=None, name=None):
    """
    Get the average score of teams in a group.

    Args:
        gid: The group id
        name: The group name
    Returns:
        The total score of the group
    """

    group_scores = get_group_scores(gid=gid, name=name)
    total_score = sum([entry['score'] for entry in group_scores])
    return int(total_score / len(group_scores)) if len(group_scores) > 0 else 0

@api.cache.memoize()
def get_all_team_scores():
    """
    Gets the score for every team in the database.

    Returns:
        A list of dictionaries with name and score
    """

    teams = api.team.get_all_teams()

    result = []
    for team in teams:
        result.append({
            "name": team['team_name'],
            "tid": team['tid'],
            "score": get_score(tid=team['tid'])
        })

    return sorted(result, key=lambda entry: entry['score'], reverse=True)

def get_all_user_scores():
    """
    Gets the score for every user in the database.

    Returns:
        A list of dictionaries with name and score
    """

    users = api.user.get_all_users()

    result = []
    for user in users:
        result.append({
            "name": user['username'],
            "score": get_score(uid=user['uid'])
        })

    return sorted(result, key=lambda entry: entry['score'], reverse=True)

@api.cache.memoize(timeout=120, fast=True)
def get_problems_by_category():
    """
    Gets the list of all problems divided into categories

    Returns:
        A dictionary of category:[problem list]
    """

    result = {cat: _get_problem_names(api.problem.get_all_problems(category=cat))
                            for cat in api.problem.get_all_categories()}

    return result

def get_team_member_stats(tid):
    members = api.team.get_team_members(tid=tid)

    return {member['username']: _get_problem_names(api.problem.get_solved_problems(uid=member['uid'])) for member in members}

#@api.cache.memoize()
def get_score_over_time(uid=None, tid=None, category=None):
    """
    Finds the score and time after each correct submission of a team or user.
    NOTE: this is slower than get_score. Do not use this for getting current score.

    Args:
        uid: the uid of the user
        tid: the tid of the user
        category: category filter
    Returns:
        A list of dictionaries containing score and time
    """

    correct_submissions = api.problem.get_submissions(uid=uid, tid=tid, category=category, correctness=True)

    result = []
    score = 0

    for submission in sorted(correct_submissions, key=lambda sub: sub["timestamp"]):
        score += api.problem.get_problem(pid=submission["pid"])["score"]
        result.append({
            "score": score,
            "time": int(submission["timestamp"].timestamp())
        })

    return result

def get_top_teams():
    """
    Finds the top teams

    Returns:
        The top teams and their scores
    """

    all_teams = api.stats.get_all_team_scores()
    return all_teams if len(all_teams) < top_teams else all_teams[:top_teams]
<|endoftext|>


<commit_before># datastore entities required by API.
# https://developers.google.com/appengine/docs/python/gettingstartedpython27/usingdatastore
# building relationships: https://developers.google.com/appengine/articles/modeling

from google.appengine.ext import db
from handlerBase import *
from google.appengine.api import memcache # import memcache

import logging

'''
	Datastore Classes (entities), prefaced by an ancestor function to make them consistent.
'''
def app_key(name = 'default'):
    return db.Key.from_path('things', name)

class Types(db.Model):
	# What type of message/pin is being placed.
	description = db.StringProperty(choices=('General Message', 'Help Needed', 'Trash Pickup'))

	@classmethod
	def by_id(cls, typeId):
		# looks up type by id
		return Comments.get_by_id(typeId, parent = app_key)

	@classmethod
	def by_descriptionType(cls,name):
		# looks up type by its description
		dt = Types.all().filter('description =', name).get()
		return dt

class GridPoints(db.Model):
	# GridPoints contains the points of the map grid.

	lat = db.FloatProperty()
	lon = db.FloatProperty()
	secondsWorked = db.FloatProperty()

	@classmethod
	def by_id(cls, gridId):
		return GridPoints.get_by_id(gridId, parent = app_key)

	@classmethod
	def by_lat(cls,name):
		latitudes = GridPoints.all().filter('lat =', name).get()
		return latitudes

	@classmethod
	def by_lon(cls,name):
		longitudes = GridPoints.all().filter('lon =', name).get()
		return longitudes

class Comments(db.Model):
	# A forum message with a particular type and timestamp, also optionally associated to a pin

	commentType = db.ReferenceProperty(Types, collection_name ='types')
	message = db.TextProperty()
	timeSent = db.DateTimeProperty(auto_now_add = True)

	@classmethod
	def by_id(cls, commentId):
		# looks up comment by id
		return Comments.get_by_id(commentId, parent = app_key)

	@classmethod
	def by_commentType(cls,name):
		# looks up comment by comment type
		ct = Comments.all().filter('commentType =', name).get()
		return ct

class Pins(db.Model):
	# Pins are latitude and longitude points on the map with a particular type and a comment associated with them.

	comment = db.ReferenceProperty(Comments, collection_name = 'comments')
	pinType = db.ReferenceProperty(Types, collection_name = 'types')

	lat = db.FloatProperty()
	lon = db.FloatProperty()

	@classmethod
	def by_id(cls, pinId):
		return Pins.get_by_id(pinId, parent = app_key)

	@classmethod
	def by_comment(cls, name):
		bc = Pins.all().filter('comment =', name).get()
		return bc

	@classmethod
	def by_type(cls, name):
		bt = Pins.all().filter('pinType =', name).get()
		return bt

	@classmethod
	def by_lat(cls,name):
		latitudes = Pins.all().filter('lat =', name).get()
		return latitudes

	@classmethod
	def by_lon(cls,name):
		longitudes = Pins.all().filter('lon =', name).get()
		return longitudes

'''
	Memcache methods that the user will call instead of direct datastore queries.
'''

'''
	Test harness for reading and writing entities in the datastore.
'''
class MakeDatastoreTest(webapp2.RequestHandler):
	def get(self):
		self.response.write("write worked")

		# Set the same parent key on each entity to ensure consistent results and add some junk data
		types = Types(parent = app_key(), description = "General Message")
		gridPoints = GridPoints(parent = app_key(), lat = 1.0, lon = 2.0, secondsWorked = 10.0)
		comments = Comments(parent = app_key(), message = 'this is a message', commentType = "needs")
		pins = Pins(parent = app_key(), lat= 1.1, lon= 1.2)

		# put the junk data in the datastore
		types.put()
		gridPoints.put()
		comments.put()
		pins.put()

class DisplayDatastoreTest(webapp2.RequestHandler):
	def get(self):
		self.response.write("check logs for read")
		# types
		logging.info("## Types ##")
		types_query = Types.all()
		types_query.ancestor(app_key())

		for types in types_query.run():
			logging.info(types.description)

		# gridPoints
		logging.info("## GridPoints ##")
		gridPoints_query = GridPoints.all()
		gridPoints_query.ancestor(app_key())

		for gridPoints in gridPoints_query.run():
			logging.info('lat= %s, lon= %s, secondsWorked=%s' %(str(gridPoints.lat), str(gridPoints.lon), str(gridPoints.secondsWorked)) )

		# comments
		logging.info("## Comments ##")
		comments_query = Comments.all()
		comments_query.ancestor(app_key())

		for comments in comments_query.run():
			logging.info(comments.message)

		# pins
		logging.info("## Pins ##")
		pins_query = Pins.all()
		pins_query.ancestor(app_key())

		for pins in pins_query.run():
			logging.info('lat= %s, lon=%s' %(str(pins.lat), str(pins.lon)) )<commit_msg>added more to the test harness and fixed some issues with the datastore methods<commit_after># datastore entities required by API.
# https://developers.google.com/appengine/docs/python/gettingstartedpython27/usingdatastore
# building relationships: https://developers.google.com/appengine/articles/modeling

from google.appengine.ext import db
from handlerBase import *
from google.appengine.api import memcache # import memcache

import logging

'''
	Datastore Classes (entities), prefaced by an ancestor function to make them consistent.
'''
def app_key(name = 'default'):
    return db.Key.from_path('things', name)

class Types(db.Model):
	# What type of message/pin is being placed.
	description = db.StringProperty(choices=('General Message', 'Help Needed', 'Trash Pickup'))

	@classmethod
	def by_id(cls, typeId):
		# looks up type by id
		return Comments.get_by_id(typeId, parent = app_key)

	@classmethod
	def by_descriptionType(cls,name):
		# looks up type by its description
		dt = Types.all().filter('description =', name).get()
		return dt

class GridPoints(db.Model):
	# GridPoints contains the points of the map grid.

	lat = db.FloatProperty()
	lon = db.FloatProperty()
	secondsWorked = db.FloatProperty()

	@classmethod
	def by_id(cls, gridId):
		return GridPoints.get_by_id(gridId, parent = app_key)

	@classmethod
	def by_lat(cls,name):
		latitudes = GridPoints.all().filter('lat =', name).get()
		return latitudes

	@classmethod
	def by_lon(cls,name):
		longitudes = GridPoints.all().filter('lon =', name).get()
		return longitudes

class Comments(db.Model):
	# A forum message with a particular type and timestamp, also optionally associated to a pin

	commentType = db.ReferenceProperty(Types, collection_name ='types')
	message = db.TextProperty()
	timeSent = db.DateTimeProperty(auto_now_add = True)

	@classmethod
	def by_id(cls, commentId):
		# looks up comment by id
		return Comments.get_by_id(commentId, parent = app_key)

	@classmethod
	def by_commentType(cls,name):
		# looks up comment by comment type
		ct = Comments.all().filter('commentType =', name).get()
		return ct

class Pins(db.Model):
	# Pins are latitude and longitude points on the map with a particular type and a comment associated with them.

	comment = db.ReferenceProperty(Comments, collection_name = 'comments')
	pinType = db.ReferenceProperty(Types, collection_name = 'pin_types')

	lat = db.FloatProperty()
	lon = db.FloatProperty()

	@classmethod
	def by_id(cls, pinId):
		return Pins.get_by_id(pinId, parent = app_key)

	@classmethod
	def by_comment(cls, name):
		bc = Pins.all().filter('comment =', name).get()
		return bc

	@classmethod
	def by_type(cls, name):
		bt = Pins.all().filter('pinType =', name).get()
		return bt

	@classmethod
	def by_lat(cls,name):
		latitudes = Pins.all().filter('lat =', name).get()
		return latitudes

	@classmethod
	def by_lon(cls,name):
		longitudes = Pins.all().filter('lon =', name).get()
		return longitudes

'''
	Memcache methods that the user will call instead of direct datastore queries.
'''
def setData(key, val):
	# simple wrapper for memcache.set, in case we need to extend it.
	memcache.set(key, val)

def getData(key):
	# wrapper for memcache get, will return none if the data wasn't found
	result = memcache.get(key)

	if not result:
		return None

	return result

'''
	Test harness for reading and writing entities in the datastore.
'''
class MakeDatastoreTest(webapp2.RequestHandler):
	def get(self):
		self.response.write("write worked")

		# Set the same parent key on each entity to ensure consistent results and add some junk data
		types = Types(parent = app_key(), description = "Help Needed")
		gridPoints = GridPoints(parent = app_key(), lat = 1.0, lon = 2.0, secondsWorked = 10.0)
		comments = Comments(parent = app_key(), message = 'this is a message')
		pins = Pins(parent = app_key(), lat= 1.1, lon= 1.2)

		# put the junk data in the datastore
		types.put()
		gridPoints.put()
		comments.put()
		pins.put()

		# test out relational modelling
		types = Types(parent = app_key(), description = "General Message")
		types.put()
		Comments(parent=app_key(), commentType = types, message='i hope this works').put()

class DisplayDatastoreTest(webapp2.RequestHandler):
	def get(self):
		self.response.write("check logs for read")
		# types
		logging.info("## Types ##")
		types_query = Types.all()
		types_query.ancestor(app_key())

		for types in types_query.run():
			logging.info(types.description)

		# gridPoints
		logging.info("## GridPoints ##")
		gridPoints_query = GridPoints.all()
		gridPoints_query.ancestor(app_key())

		for gridPoints in gridPoints_query.run():
			logging.info('lat= %s, lon= %s, secondsWorked=%s' %(str(gridPoints.lat), str(gridPoints.lon), str(gridPoints.secondsWorked)) )

		# comments
		logging.info("## Comments ##")
		comments_query = Comments.all()
		comments_query.ancestor(app_key())

		for comments in comments_query.run():
			logging.info(comments.message)

		# pins
		logging.info("## Pins ##")
		pins_query = Pins.all()
		pins_query.ancestor(app_key())

		for pins in pins_query.run():
			logging.info('lat= %s, lon=%s' %(str(pins.lat), str(pins.lon)) )<|endoftext|>


<commit_before>import numpy
from matplotlib import pyplot
from matplotlib.backends.backend_pdf import PdfPages
import os, sys
import argparse

"""Read the program similarity result files and plot histograms"""

def parse_result_file(result_file):
	"""
	file format:
	path_to_dotA:
	path_to_similar_dot1 , score
	...
	path_to_similar_dot5 , score

	path_to_dotB:
	...
	"""
	avg_score_vector = []

	count = 0
	score = 0.0
	with open(result_file, "r") as fi:
		for line in fi:
			line = line.rstrip('\n')
			linarr = line.split(" , ")
			if linarr[0][-3:]=="dot":
				count += 1
				score += float(linarr[1])
				if count==5:
					avg_score_vector.append(score/count)
					count = 0
					score = 0.0
	return avg_score_vector

def show_improvement(avg_score_vector_nc, avg_score_vector_c):
	total = 0.0
	assert len(avg_score_vector_nc)==len(avg_score_vector_c), "Should have the same number of methods with or without clustering."
	for i in range(len(avg_score_vector_nc)):
		assert avg_score_vector_nc[i]<=avg_score_vector_c[i], "Clustering cannot degrade the performance of similar program identification."
		total += avg_score_vector_c[i] - avg_score_vector_nc[i]
	print("Total similarity score improvement: {0}.".format(total))
	print("Average similarity score improvement per method: {0}.".format(total/len(avg_score_vector_c)))

def plot_hist(x, xlabel, y, ylabel, fig_file):
	bins = numpy.linspace(0.0, 2.0, 100)
	#pyplot.hist(x, bins, alpha=0.5, label=xlabel)
	#pyplot.hist(y, bins, alpha=0.5, label=ylabel)
	data = numpy.vstack([x, y]).T
	pyplot.hist(data, bins, alpha=0.7, label=[xlabel, ylabel])
	pyplot.legend(loc="upper right")
	#pyplot.show()
	pp = PdfPages(fig_file+".pdf")
	pyplot.savefig(pp, format='pdf')
	pp.close()

def main():

	parser = argparse.ArgumentParser()
	parser.add_argument("-nc", "--nocluster", required=True, type=str, help="path to the result file without relabeling")
	parser.add_argument("-c", "--cluster", required=True, type=str, help="path to the result file with relabeling")
	parser.add_argument("-f", "--file", required=True, type=str, help="name of the pdf figure (without the .pdf extension")
	args = parser.parse_args()

	score1 = parse_result_file(args.nocluster)
	score2 = parse_result_file(args.cluster)

	plot_hist(score1, "no cluster", score2, "cluster", args.file)

	show_improvement(score1, score2)

if __name__ == "__main__":
	main()
<commit_msg>commented out an assertion<commit_after>import numpy
from matplotlib import pyplot
from matplotlib.backends.backend_pdf import PdfPages
import os, sys
import argparse

"""Read the program similarity result files and plot histograms"""

def parse_result_file(result_file):
	"""
	file format:
	path_to_dotA:
	path_to_similar_dot1 , score
	...
	path_to_similar_dot5 , score

	path_to_dotB:
	...
	"""
	avg_score_vector = []

	count = 0
	score = 0.0
	with open(result_file, "r") as fi:
		for line in fi:
			line = line.rstrip('\n')
			linarr = line.split(" , ")
			if linarr[0][-3:]=="dot":
				count += 1
				score += float(linarr[1])
				if count==5:
					avg_score_vector.append(score/count)
					count = 0
					score = 0.0
	return avg_score_vector

def show_improvement(avg_score_vector_nc, avg_score_vector_c):
	total = 0.0
	assert len(avg_score_vector_nc)==len(avg_score_vector_c), "Should have the same number of methods with or without clustering."
	for i in range(len(avg_score_vector_nc)):
		#assert avg_score_vector_nc[i]<=avg_score_vector_c[i], "Clustering cannot degrade the performance of similar program identification."
		total += avg_score_vector_c[i] - avg_score_vector_nc[i]
	print("Total similarity score improvement: {0}.".format(total))
	print("Average similarity score improvement per method: {0}.".format(total/len(avg_score_vector_c)))

def plot_hist(x, xlabel, y, ylabel, fig_file):
	bins = numpy.linspace(0.0, 2.0, 100)
	#pyplot.hist(x, bins, alpha=0.5, label=xlabel)
	#pyplot.hist(y, bins, alpha=0.5, label=ylabel)
	data = numpy.vstack([x, y]).T
	pyplot.hist(data, bins, alpha=0.7, label=[xlabel, ylabel])
	pyplot.legend(loc="upper right")
	#pyplot.show()
	pp = PdfPages(fig_file+".pdf")
	pyplot.savefig(pp, format='pdf')
	pp.close()

def main():

	parser = argparse.ArgumentParser()
	parser.add_argument("-nc", "--nocluster", required=True, type=str, help="path to the result file without relabeling")
	parser.add_argument("-c", "--cluster", required=True, type=str, help="path to the result file with relabeling")
	parser.add_argument("-f", "--file", required=True, type=str, help="name of the pdf figure (without the .pdf extension")
	args = parser.parse_args()

	score1 = parse_result_file(args.nocluster)
	score2 = parse_result_file(args.cluster)

	plot_hist(score1, "no cluster", score2, "cluster", args.file)

	show_improvement(score1, score2)

if __name__ == "__main__":
	main()
<|endoftext|>


<commit_before># Copyright (C) 2015 Sam Parkinson
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import re
import sys
import json

HELP = '''Usage:

    python pluginify.py (file)

or

    python pluginify.py (file) > plugin.json

Converts an RTP (readable TurtleBlocks plugin) file into a JSON file
to load into Turtle Blocks JS.  For more information, run
`python pluginify.py syntax`
'''

SYNTAX = '''

In an RST file, new blocks are not defined with braces (as is typical of
Javascript); rather they start whenever you add //* *// and their scope is
until the next block starts (or the end of file in the case that it is
the last block).

To add a comment, just type //* comment *// followed by your multi-
line comment. pluginify ignores comments, i.e., they are not added in
the JSON plugin.

Example:
//* comment *// Your single or multi line comment here...

Global variables are defined in a section //* globals *//.

Example:
//* globals *//
var calories = 0;
var protein = 0;
var carbohydrate = 0;
var fiber = 0;
var fat = 0;

Define all the global variables under the section
//* block-globals *//.

These definitions will be added to all the code blocks in the created
JSON output. Note that these "globals" included in each block but are
local in context to each block.

Example: You can define a common API Key to be used by all blocks.
//* block-globals *//
var mashapeKey = '(keycode)'; in globals.

You can also declare global variables specific to argument blocks that
get applied to a set of similar blocks, e.g., the variables under
//* arg-globals *// will be added to all the arg blocks.

Example:
//* arg-globals *//
var block = blocks.blockList[blk];
var connections = block.connections;

You can declare functions for parameter blocks to be evaluated when
the block labels are updated with the //* parameter:(blockname) *//
tag.

Example:
//* parameter:loudness *//
if (mic == null) {errorMsg("The microphone is not available.");
    value = 0;
} else {
    value = Math.round(mic.getLevel() * 1000);
}

To define a block you need to type: //* block:(blockname) *//

Example:
//* block:power *//
var block = new ProtoBlock('power');
block.palette = palettes.dict['maths'];
blocks.protoBlockDict['power'] = block;
block.twoArgMathBlock();
block.defaults.push(10, 2);
block.staticLabels.push('power', 'base', 'exp.');

You should also define a setter for a parameter block (if appropriate).
You can do this using the setter setion:

//* setter:myValue *//
myValue = value;
updateDisplayOfMyValue();

Graphical elements (icons, colors) are defined in the own sections:

Palette icons are defined as //* palette-icon:(palette name) *//

Example:
//* palette-icon:food *//
<svg ...> ... </svg>

Similarly for block colors:

Example:
//* palette-fill:food *// #FFFFFF
//* palette-stroke:food *// #A0A0A0
//* palette-highlight:food *// #D5D5D5
'''

def clear():
    global NAMES, JS_TYPES, IMAGES
    NAMES = {
        'flow': 'FLOWPLUGINS',
        'arg': 'ARGPLUGINS',
        'block': 'BLOCKPLUGINS',
        'parameter': 'PARAMETERPLUGINS',
        'setter': 'SETTERPLUGINS',
        'palette-icon': 'PALETTEPLUGINS',
        'palette-fill': 'PALETTEFILLCOLORS',
        'palette-stroke': 'PALETTESTROKECOLORS',
        'palette-highlight': 'PALETTEHIGHLIGHTCOLORS',
        'palette-stroke-highlight': 'HIGHLIGHTSTROKECOLORS'}
    JS_TYPES = ('flow', 'arg', 'block', 'parameter', 'setter', 'onload', 'onstart', 'onstop')
    # 'blkName': 'imageData',
    IMAGES = {}

def pluginify(data):
    clear()
    sections_list = data.split('//*')
    sections_pairs = []
    specific_globals = {x: '' for x in JS_TYPES}
    globals_ = None
    for section in sections_list:
        match = re.match('(.*)\*\/\/([^\0]*)', section.strip())
        if match:
            if match.group(1).strip() == 'globals':
                globals_ = match.group(2).strip()
            elif match.group(1).strip().endswith('-globals'):
                type_, _ = match.group(1).strip().split('-')
                specific_globals[type_] = specific_globals[type_] + \
                    match.group(2).strip()
            elif match.group(1).strip() == 'comment':
                continue
            else:
                sections_pairs.append((match.group(1).strip(),
                                       match.group(2).strip()))

    outp = {}
    if globals_ is not None:
        outp['GLOBALS'] = globals_.replace('\n', '').replace('var ', '')

    for key, value in sections_pairs:
        if len(key.split(':')) != 2:
            raise ValueError('Section names should have 1 colon (type:name)')
        type_, name = key.split(':')

        if type_ in JS_TYPES:
            value = specific_globals[type_] + value
        value = value.replace('\n', '')

        if type_ in NAMES:
            type_ = NAMES[type_]
            if type_ not in outp:
                outp[type_] = {}
            outp[type_][name] = value

        if type_ == 'image':
            # TODO: Detect if its png
            IMAGES[name] = 'data:image/svg+xml;utf8,' + value

    if IMAGES:
        outp['IMAGES'] = IMAGES

    return json.dumps(outp, indent=4)

if __name__ == '__main__':
    if len(sys.argv) != 2:
        print HELP
    elif sys.argv[1] in ('help', '-h', '--help'):
        print HELP
    elif sys.argv[1] == 'syntax':
        print SYNTAX
    else:
        with open(sys.argv[1]) as f:
            data = f.read()
        print pluginify(data)
<commit_msg>add missing constants<commit_after># Copyright (C) 2015 Sam Parkinson
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import re
import sys
import json

HELP = '''Usage:

    python pluginify.py (file)

or

    python pluginify.py (file) > plugin.json

Converts an RTP (readable TurtleBlocks plugin) file into a JSON file
to load into Turtle Blocks JS.  For more information, run
`python pluginify.py syntax`
'''

SYNTAX = '''

In an RST file, new blocks are not defined with braces (as is typical of
Javascript); rather they start whenever you add //* *// and their scope is
until the next block starts (or the end of file in the case that it is
the last block).

To add a comment, just type //* comment *// followed by your multi-
line comment. pluginify ignores comments, i.e., they are not added in
the JSON plugin.

Example:
//* comment *// Your single or multi line comment here...

Global variables are defined in a section //* globals *//.

Example:
//* globals *//
var calories = 0;
var protein = 0;
var carbohydrate = 0;
var fiber = 0;
var fat = 0;

Define all the global variables under the section
//* block-globals *//.

These definitions will be added to all the code blocks in the created
JSON output. Note that these "globals" included in each block but are
local in context to each block.

Example: You can define a common API Key to be used by all blocks.
//* block-globals *//
var mashapeKey = '(keycode)'; in globals.

You can also declare global variables specific to argument blocks that
get applied to a set of similar blocks, e.g., the variables under
//* arg-globals *// will be added to all the arg blocks.

Example:
//* arg-globals *//
var block = blocks.blockList[blk];
var connections = block.connections;

You can declare functions for parameter blocks to be evaluated when
the block labels are updated with the //* parameter:(blockname) *//
tag.

Example:
//* parameter:loudness *//
if (mic == null) {errorMsg("The microphone is not available.");
    value = 0;
} else {
    value = Math.round(mic.getLevel() * 1000);
}

To define a block you need to type: //* block:(blockname) *//

Example:
//* block:power *//
var block = new ProtoBlock('power');
block.palette = palettes.dict['maths'];
blocks.protoBlockDict['power'] = block;
block.twoArgMathBlock();
block.defaults.push(10, 2);
block.staticLabels.push('power', 'base', 'exp.');

You should also define a setter for a parameter block (if appropriate).
You can do this using the setter setion:

//* setter:myValue *//
myValue = value;
updateDisplayOfMyValue();

Graphical elements (icons, colors) are defined in the own sections:

Palette icons are defined as //* palette-icon:(palette name) *//

Example:
//* palette-icon:food *//
<svg ...> ... </svg>

Similarly for block colors:

Example:
//* palette-fill:food *// #FFFFFF
//* palette-stroke:food *// #A0A0A0
//* palette-highlight:food *// #D5D5D5

Plugins can specify code to be executed on load, on start, and on stop.
Example:

//* onload:foo *//
your code here...

NOTE: name of on load, on start, and on stop sections must match the
name of one of the plugin blocks.
'''

def clear():
    global NAMES, JS_TYPES, IMAGES
    NAMES = {
        'flow': 'FLOWPLUGINS',
        'arg': 'ARGPLUGINS',
        'block': 'BLOCKPLUGINS',
        'parameter': 'PARAMETERPLUGINS',
        'setter': 'SETTERPLUGINS',
        'onload': 'ONLOAD',
        'onstart': 'ONSTART',
        'onstop': 'ONSTOP',
        'palette-icon': 'PALETTEPLUGINS',
        'palette-fill': 'PALETTEFILLCOLORS',
        'palette-stroke': 'PALETTESTROKECOLORS',
        'palette-highlight': 'PALETTEHIGHLIGHTCOLORS',
        'palette-stroke-highlight': 'HIGHLIGHTSTROKECOLORS'}
    JS_TYPES = ('flow', 'arg', 'block', 'parameter', 'setter', 'onload', 'onstart', 'onstop')
    # 'blkName': 'imageData',
    IMAGES = {}

def pluginify(data):
    clear()
    sections_list = data.split('//*')
    sections_pairs = []
    specific_globals = {x: '' for x in JS_TYPES}
    globals_ = None
    for section in sections_list:
        match = re.match('(.*)\*\/\/([^\0]*)', section.strip())
        if match:
            if match.group(1).strip() == 'globals':
                globals_ = match.group(2).strip()
            elif match.group(1).strip().endswith('-globals'):
                type_, _ = match.group(1).strip().split('-')
                specific_globals[type_] = specific_globals[type_] + \
                    match.group(2).strip()
            elif match.group(1).strip() == 'comment':
                continue
            else:
                sections_pairs.append((match.group(1).strip(),
                                       match.group(2).strip()))

    outp = {}
    if globals_ is not None:
        outp['GLOBALS'] = globals_.replace('\n', '').replace('var ', '')

    for key, value in sections_pairs:
        if len(key.split(':')) != 2:
            raise ValueError('Section names should have 1 colon (type:name)')
        type_, name = key.split(':')

        if type_ in JS_TYPES:
            value = specific_globals[type_] + value
        value = value.replace('\n', '')

        if type_ in NAMES:
            type_ = NAMES[type_]
            if type_ not in outp:
                outp[type_] = {}
            outp[type_][name] = value

        if type_ == 'image':
            # TODO: Detect if its png
            IMAGES[name] = 'data:image/svg+xml;utf8,' + value

    if IMAGES:
        outp['IMAGES'] = IMAGES

    return json.dumps(outp, indent=4)

if __name__ == '__main__':
    if len(sys.argv) != 2:
        print HELP
    elif sys.argv[1] in ('help', '-h', '--help'):
        print HELP
    elif sys.argv[1] == 'syntax':
        print SYNTAX
    else:
        with open(sys.argv[1]) as f:
            data = f.read()
        print pluginify(data)
<|endoftext|>


<commit_before>from itertools import groupby
import datetime
import os.path
import bisect
import iso8601

from flask import url_for

MIN_YEAR = 2006

ICONS = {
    "member": "bill-introduced.png",
    "committee": "committee-discussion.png",
    "house": "house.png",
    "president": "signed-by-president.png",
    "unknown": "bill-introduced.png",
}

def get_location(event):
    if event.get('type') in ['bill-signed', 'bill-act-commenced', 'bill-enacted']:
        return {
            'name': 'Office of the President',
            'class': 'president',
        }

    if event.get('house'):
        return {
            'name': event['house']['name'],
            'class': event['house']['short_name'],
        }

    if event.get('committee'):
        if 'house' in event['committee']:
            return {
                'name': event['committee']['house']['name'],
                'class': event['committee']['house']['short_name'],
            }

        return {
            'name': event['committee']['name'],
            'url': url_for('committee_detail', committee_id=event['committee']['id']),
            'class': '',
        }

    return {'name': 'Unknown', 'class': ''}

def get_agent(event, bill):
    info = None

    if event.get('type') in ['bill-signed', 'bill-act-commenced', 'bill-enacted']:
        info = {
            'name': 'The President',
            'type': 'president',
        }

    elif event.get('type') == 'bill-introduced':
        info = {
            'name': bill['introduced_by'] or (bill.get('place_of_introduction') or {}).get('name'),
            'type': 'member',
        }

    elif event.get('member'):
        info = {
            'name': event['member']['name'],
            'type': 'member',
            'url': url_for('member', member_id=event['member']['id'])
        }

    elif event.get('committee'):
        info = {
            'name': event['committee']['name'],
            'type': 'committee',
            'url': url_for('committee_detail', committee_id=event['committee']['id'])
        }

    elif event.get('house'):
        info = {
            'name': event['house']['name'],
            'type': 'house',
        }
    else:
        info = {'name': 'Unknown', 'type': 'unknown'}

    info['icon'] = ICONS[info['type']]

    return info

def bill_history(bill):
    """ Work out the history of a bill and return a description of it. """
    history = []

    events = bill.get('events', [])
    events.sort(key=lambda e: [
        iso8601.parse_date(e['date']),
        get_location(e),
        get_agent(e, bill)])

    for location, location_events in groupby(events, get_location):
        location_history = []

        for agent, agent_events in groupby(location_events, lambda e: get_agent(e, bill)):
            info = {'events': list(agent_events)}
            info.update(agent)
            location_history.append(info)

        info = {'events': location_history}
        info.update(location)
        history.append(info)

    history = hansard_linking(history)
    return history

def match_title(event_title):
    """
    Match bill title against the following possible titles
    """
    bill_titles = [
        "Bill passed by National Assembly",
        "Bill passed by both Houses",
        "Bill revived on this date",
        "The NCOP rescinded",
        "Bill remitted",
    ]
    for title in bill_titles:
        if title in event_title:
            return True
    return False

def match_dates(hansard_date, event_date):
    hansard_iso_date = iso8601.parse_date(hansard_date)
    event_iso_date = iso8601.parse_date(event_date)
    if hansard_iso_date.date() == event_iso_date.date():
        return True
    return False

def hansard_linking(bill_history):
    """
    We need to link certain bill events to hansards
    Hansrds will always be linked to a house (NA or NCOP)
    The Date of the bill event and the hansard will be the same.
    Bill Titles we are looking for:
    * Bill passed by National Assembly
    * Bill passed by both Houses
    * Bill revived on this date
    * The NCOP rescinded
    * Bill remitted
    """

    for class_history in bill_history:
        for event_history in class_history["events"]:
            if event_history["type"] == "house":
                for event in event_history["events"]:
                    if event["type"] == "plenary":
                        for bill_event in event_history["events"]:
                            if match_title(bill_event["title"]) and match_dates(
                                event["date"], bill_event["date"]
                            ):
                                bill_event["hansard"] = {"id": event["id"]}
    return bill_history

def count_parliamentary_days(date_from, date_to):
    """ Count the number of parliamentary days between two dates, inclusive.
    """
    i = bisect.bisect(PARLIAMENTARY_DAYS, date_from)
    j = bisect.bisect(PARLIAMENTARY_DAYS, date_to)
    return j - i + 1

def load_parliamentary_days():
    """ Load the dates when parliament sat from data/parliament-sitting-days.txt

    This file can be updated from a spreadsheet using bin/load_parliamentary_days.py
    """
    with open(os.path.join(os.path.dirname(__file__), "../data/parliament-sitting-days.txt"), "r") as f:
        lines = f.readlines()
        dates = [datetime.date(*(int(x) for x in d.split("-"))) for d in lines]
        return sorted(dates)

PARLIAMENTARY_DAYS = load_parliamentary_days()
<commit_msg>Add some more documentation<commit_after>from itertools import groupby
import datetime
import os.path
import bisect
import iso8601

from flask import url_for

MIN_YEAR = 2006

ICONS = {
    "member": "bill-introduced.png",
    "committee": "committee-discussion.png",
    "house": "house.png",
    "president": "signed-by-president.png",
    "unknown": "bill-introduced.png",
}

def get_location(event):
    if event.get('type') in ['bill-signed', 'bill-act-commenced', 'bill-enacted']:
        return {
            'name': 'Office of the President',
            'class': 'president',
        }

    if event.get('house'):
        return {
            'name': event['house']['name'],
            'class': event['house']['short_name'],
        }

    if event.get('committee'):
        if 'house' in event['committee']:
            return {
                'name': event['committee']['house']['name'],
                'class': event['committee']['house']['short_name'],
            }

        return {
            'name': event['committee']['name'],
            'url': url_for('committee_detail', committee_id=event['committee']['id']),
            'class': '',
        }

    return {'name': 'Unknown', 'class': ''}

def get_agent(event, bill):
    info = None

    if event.get('type') in ['bill-signed', 'bill-act-commenced', 'bill-enacted']:
        info = {
            'name': 'The President',
            'type': 'president',
        }

    elif event.get('type') == 'bill-introduced':
        info = {
            'name': bill['introduced_by'] or (bill.get('place_of_introduction') or {}).get('name'),
            'type': 'member',
        }

    elif event.get('member'):
        info = {
            'name': event['member']['name'],
            'type': 'member',
            'url': url_for('member', member_id=event['member']['id'])
        }

    elif event.get('committee'):
        info = {
            'name': event['committee']['name'],
            'type': 'committee',
            'url': url_for('committee_detail', committee_id=event['committee']['id'])
        }

    elif event.get('house'):
        info = {
            'name': event['house']['name'],
            'type': 'house',
        }
    else:
        info = {'name': 'Unknown', 'type': 'unknown'}

    info['icon'] = ICONS[info['type']]

    return info

def bill_history(bill):
    """ Work out the history of a bill and return a description of it. """
    history = []

    events = bill.get('events', [])
    events.sort(key=lambda e: [
        iso8601.parse_date(e['date']),
        get_location(e),
        get_agent(e, bill)])

    for location, location_events in groupby(events, get_location):
        location_history = []

        for agent, agent_events in groupby(location_events, lambda e: get_agent(e, bill)):
            info = {'events': list(agent_events)}
            info.update(agent)
            location_history.append(info)

        info = {'events': location_history}
        info.update(location)
        history.append(info)

    history = hansard_linking(history)
    return history

def match_title(event_title):
    """
    Match bill title against the following possible titles
    """
    bill_titles = [
        "Bill passed by National Assembly",
        "Bill passed by both Houses",
        "Bill revived on this date",
        "The NCOP rescinded",
        "Bill remitted",
    ]
    for title in bill_titles:
        if title in event_title:
            return True
    return False

def match_dates(hansard_date, event_date):
    hansard_iso_date = iso8601.parse_date(hansard_date)
    event_iso_date = iso8601.parse_date(event_date)
    if hansard_iso_date.date() == event_iso_date.date():
        return True
    return False

def hansard_linking(bill_history):
    """
    We need to link certain bill events to hansards
    Hansrds will always be linked to a house (NA or NCOP)
    The Date of the bill event and the hansard will be the same.
    Bill Titles we are looking for:
    * Bill passed by National Assembly
    * Bill passed by both Houses
    * Bill revived on this date
    * The NCOP rescinded
    * Bill remitted
    If the event(bill_passed etc) is matched, a new dict is created with the matching hansard id.
    The Hansard event is not modified.
    """

    for class_history in bill_history:
        for event_history in class_history["events"]:
            if event_history["type"] == "house":
                for event in event_history["events"]:
                    if event["type"] == "plenary":
                        for bill_event in event_history["events"]:
                            if match_title(bill_event["title"]) and match_dates(
                                event["date"], bill_event["date"]
                            ):
                                bill_event["hansard"] = {"id": event["id"]}
    return bill_history

def count_parliamentary_days(date_from, date_to):
    """ Count the number of parliamentary days between two dates, inclusive.
    """
    i = bisect.bisect(PARLIAMENTARY_DAYS, date_from)
    j = bisect.bisect(PARLIAMENTARY_DAYS, date_to)
    return j - i + 1

def load_parliamentary_days():
    """ Load the dates when parliament sat from data/parliament-sitting-days.txt

    This file can be updated from a spreadsheet using bin/load_parliamentary_days.py
    """
    with open(os.path.join(os.path.dirname(__file__), "../data/parliament-sitting-days.txt"), "r") as f:
        lines = f.readlines()
        dates = [datetime.date(*(int(x) for x in d.split("-"))) for d in lines]
        return sorted(dates)

PARLIAMENTARY_DAYS = load_parliamentary_days()
<|endoftext|>