<commit_before>__author__ = 'Raghav Sidhanti'

import device
import message

from inout import StdIn
from inout import StdOut
from configuration import Conf
from configuration import ConfUtil

# defines all the available fog commands.
# every command defines default pre and post steps.
# executes appropriate methods on services

class CommandInvoker(object):

    def invoke(self, command):
        if command is None:
            return

        # if in new state, can execute only init, help and invalid
        if not ConfUtil.exists_home():
            if not isinstance(command, Help) and not isinstance(command, Init) and not isinstance(command, Invalid):
                StdOut.display(msg=message.get(message.MISSING_HOME))
                return

        if command.valid():
            # if in ready state, all commands can be executed
            command.execute()

class CommandParser(object):

    @staticmethod
    def __get_command(inputs):
        args = inputs[1:]
        # create instance for drive and return
        return {
            Branch.name(): lambda: Branch(args),
            Checkout.name(): lambda: Checkout(args),
            Help.name(): lambda: Help(inputs),
            Init.name(): lambda: Init(args),
            Remote.name(): lambda: Remote(args),
            Pull.name(): lambda: Pull(args)
        }.get(inputs[0], lambda: Invalid(args))()

    def parse(self, args=['invalid']):
        # clean the args
        inputs = []
        for arg in args:
            if arg:
                inputs.append(arg.strip())
        # get command object
        return self.__get_command(inputs)

class FogCommand(object):
    _args = []

    def __init__(self, args=[]):
        self._args = args

    def execute(self, **kwargs):
        pass

    def help(self):
        pass

    def valid(self):
        pass

    @staticmethod
    def name():
        pass

class Init(FogCommand):

    @staticmethod
    def name():
        return 'init'

    def valid(self):
        if ConfUtil.exists_home():
            if StdIn.prompt_yes(message.HOME_EXISTS):
                return True
            else:
                return False

        return True

    def execute(self, **kwargs):
        # create home and files
        ConfUtil.remove_home()
        ConfUtil.create_home()

class Checkout(FogCommand):

    @staticmethod
    def name():
        return 'checkout'

    def valid(self):
        # check argument size
        if len(self._args) != 1:
            StdOut.display(msg=message.get(message.INVALID_ARGS))
            return False

        drive_name = self._args[0]
        # reject invalid drive
        if not ConfUtil.valid_drive(drive_name):
            StdOut.display(msg=message.get(message.INVALID_DRIVE_NAME, drive=drive_name))
            return False

        return True

    def execute(self, **kwargs):
        drive_name = self._args[0]
        ConfUtil.checkout(drive_name)

class Branch(FogCommand):

    @staticmethod
    def name():
        return 'branch'

    def valid(self):
        return True

    def execute(self, **kwargs):
        # read checkout file
        checkout = ConfUtil.get_checkout()

        # read branches and compare with checkout
        for name in Conf.drives.keys():
            mark = message.NO
            if name == checkout:
                mark = message.YES
            StdOut.display(msg=message.get(message.STATUS, mark=mark, drive=name))

class Remote(FogCommand):

    @staticmethod
    def name():
        return 'remote'

    def __get_remote(self, cmd):
        return {
            'remote': lambda: RemoteList(self._args),
            'remote add': lambda: RemoteAdd(self._args[1:]),
            'remote rm': lambda: RemoteRm(self._args[1:])
        }.get(cmd, lambda: Invalid(self._args))()

    def execute(self, **kwargs):
        cmd = ['invalid']
        if len(self._args) == 0:
            cmd = [self.name()]
        elif len(self._args) == 2:
            cmd = [self.name(), ' ', self._args[0]]

        remote = self.__get_remote(''.join(cmd))
        remote.execute(**kwargs)

class RemoteList(Remote):

    def valid(self):
        return True

    def execute(self, **kwargs):
        # find all the drive config
        for name, drive in Conf.drives.items():
            mark = message.NO
            if ConfUtil.exists_drive(name):
                mark = message.YES
            StdOut.display(msg=message.get(message.STATUS, mark=mark, drive=name))

class RemoteAdd(FogCommand):

    def valid(self):
        # check argument size
        if len(self._args) != 1:
            StdOut.display(msg=message.get(message.INVALID_ARGS))
            return False

        drive_name = self._args[0]
        # reject invalid drive
        if not ConfUtil.valid_drive(drive_name):
            StdOut.display(msg=message.get(message.INVALID_DRIVE_NAME, drive=drive_name))
            return False

        # check if remote exists
        if ConfUtil.exists_drive(drive_name):
            StdOut.display(msg=message.get(message.REMOTE_EXISTS, drive=drive_name))
            return False

        return True

    def execute(self, **kwargs):
        drive_name = self._args[0]
        drive = device.get_drive(drive_name)
        if drive is not None:
            # create configs home
            ConfUtil.create_drive(drive_name)
            # create remote
            if drive.open(**kwargs):
                drive.close(**kwargs)
            else:
                # don't create home if auth fails
                ConfUtil.remove_drive(drive_name)
        else:
            StdOut.display(msg=message.get(message.MISSING_IMPLEMENTATION, drive=drive_name))

class RemoteRm(FogCommand):

    def valid(self):
        # check argument size
        if len(self._args) != 1:
            StdOut.display(msg=message.get(message.INVALID_ARGS))
            return False

        drive_name = self._args[0]
        # reject invalid drive
        if not ConfUtil.valid_drive(drive_name):
            StdOut.display(msg=message.get(message.INVALID_DRIVE_NAME, drive=drive_name))
            return False

        # check if remote exists
        if not ConfUtil.exists_drive(drive_name):
            StdOut.display(msg=message.get(message.MISSING_REMOTE, drive=drive_name))
            return False

        return True

    def execute(self, **kwargs):
        drive_name = self._args[0]
        ConfUtil.remove_drive(drive_name)

class Pull(FogCommand):
    @staticmethod
    def name():
        return 'pull'

    def valid(self):
        if len(self._args) != 1 and len(self._args) != 2:
            return False

        # make sure drive has been checked out
        checkout = ConfUtil.get_checkout()
        if not checkout:
            StdOut.display(msg=message.get(message.MISSING_CHECKOUT))
            return False

        # make sure drive is tracked
        if not ConfUtil.exists_drive(checkout):
            StdOut.display(msg=message.get(message.MISSING_REMOTE, drive=checkout))
            return False

        return True

    def execute(self, **kwargs):
        # validate inputs
        src = self._args[0]
        dst = src
        if len(self._args) == 2:
            dst = self._args[1]

        drive = device.get_active_drive()
        # open connection
        if drive.open():
            # pull
            drive.download(src=src, dst=dst)
            #close
            drive.close()

class Invalid(FogCommand):
    def execute(self, **kwargs):
        StdOut.display(msg=message.get(message.INVALID_ARGS))

class Help(FogCommand):
    @staticmethod
    def name():
        return 'help'

    def valid(self):
        return True

    def execute(self, **kwargs):
        msgs = [
            'usage: fog <command> [<args>]',
            'Commonly used fog commands are:',
            'branch         List available drives',
            'checkout       Checkout a drive',
            'init           Create an empty fog directory or reinitialize an existing one',
            'pull           Download files from remote drive',
            'push           Upload a file to remote drive',
            'remote         List all the drives being tracked',
            'remote add     Track a remote drive',
            'remote rm      Un-track a remote drive',
            'rm             Move a file on remote drive to trash'
        ]

        for idx in range(0, 2, 1):
            StdOut.display(msg=msgs[idx])

        for idx in range(2, len(msgs), 1):
            StdOut.display(template=StdOut.IND2, msg=msgs[idx])
<commit_msg>#18: Default validation code.<commit_after>__author__ = 'Raghav Sidhanti'

import device
import message

from inout import StdIn
from inout import StdOut
from configuration import Conf
from configuration import ConfUtil

# defines all the available fog commands.
# every command defines default pre and post steps.
# executes appropriate methods on services

class CommandInvoker(object):

    def invoke(self, command):
        if command is None:
            return

        # if in new state, can execute only init, help and invalid
        if not ConfUtil.exists_home():
            if not isinstance(command, Help) and not isinstance(command, Init) and not isinstance(command, Invalid):
                StdOut.display(msg=message.get(message.MISSING_HOME))
                return

        if command.valid():
            # if in ready state, all commands can be executed
            command.execute()

class CommandParser(object):

    @staticmethod
    def __get_command(inputs):
        args = inputs[1:]
        # create instance for drive and return
        return {
            Branch.name(): lambda: Branch(args),
            Checkout.name(): lambda: Checkout(args),
            Help.name(): lambda: Help(inputs),
            Init.name(): lambda: Init(args),
            Remote.name(): lambda: Remote(args),
            Pull.name(): lambda: Pull(args)
        }.get(inputs[0], lambda: Invalid(args))()

    def parse(self, args=['invalid']):
        # clean the args
        inputs = []
        for arg in args:
            if arg:
                inputs.append(arg.strip())
        # get command object
        return self.__get_command(inputs)

class FogCommand(object):
    _args = []

    def __init__(self, args=[]):
        self._args = args

    def execute(self, **kwargs):
        pass

    def help(self):
        pass

    def valid(self):
        return True

    @staticmethod
    def name():
        pass

class Init(FogCommand):

    @staticmethod
    def name():
        return 'init'

    def valid(self):
        if ConfUtil.exists_home():
            if StdIn.prompt_yes(message.HOME_EXISTS):
                return True
            else:
                return False

        return True

    def execute(self, **kwargs):
        # create home and files
        ConfUtil.remove_home()
        ConfUtil.create_home()

class Checkout(FogCommand):

    @staticmethod
    def name():
        return 'checkout'

    def valid(self):
        # check argument size
        if len(self._args) != 1:
            StdOut.display(msg=message.get(message.INVALID_ARGS))
            return False

        drive_name = self._args[0]
        # reject invalid drive
        if not ConfUtil.valid_drive(drive_name):
            StdOut.display(msg=message.get(message.INVALID_DRIVE_NAME, drive=drive_name))
            return False

        return True

    def execute(self, **kwargs):
        drive_name = self._args[0]
        ConfUtil.checkout(drive_name)

class Branch(FogCommand):

    @staticmethod
    def name():
        return 'branch'

    def valid(self):
        return True

    def execute(self, **kwargs):
        # read checkout file
        checkout = ConfUtil.get_checkout()

        # read branches and compare with checkout
        for name in Conf.drives.keys():
            mark = message.NO
            if name == checkout:
                mark = message.YES
            StdOut.display(msg=message.get(message.STATUS, mark=mark, drive=name))

class Remote(FogCommand):

    @staticmethod
    def name():
        return 'remote'

    def __get_remote(self, cmd):
        return {
            'remote': lambda: RemoteList(self._args),
            'remote add': lambda: RemoteAdd(self._args[1:]),
            'remote rm': lambda: RemoteRm(self._args[1:])
        }.get(cmd, lambda: Invalid(self._args))()

    def execute(self, **kwargs):
        cmd = ['invalid']
        if len(self._args) == 0:
            cmd = [self.name()]
        elif len(self._args) == 2:
            cmd = [self.name(), ' ', self._args[0]]

        remote = self.__get_remote(''.join(cmd))
        remote.execute(**kwargs)

class RemoteList(Remote):

    def valid(self):
        return True

    def execute(self, **kwargs):
        # find all the drive config
        for name, drive in Conf.drives.items():
            mark = message.NO
            if ConfUtil.exists_drive(name):
                mark = message.YES
            StdOut.display(msg=message.get(message.STATUS, mark=mark, drive=name))

class RemoteAdd(FogCommand):

    def valid(self):
        # check argument size
        if len(self._args) != 1:
            StdOut.display(msg=message.get(message.INVALID_ARGS))
            return False

        drive_name = self._args[0]
        # reject invalid drive
        if not ConfUtil.valid_drive(drive_name):
            StdOut.display(msg=message.get(message.INVALID_DRIVE_NAME, drive=drive_name))
            return False

        # check if remote exists
        if ConfUtil.exists_drive(drive_name):
            StdOut.display(msg=message.get(message.REMOTE_EXISTS, drive=drive_name))
            return False

        return True

    def execute(self, **kwargs):
        drive_name = self._args[0]
        drive = device.get_drive(drive_name)
        if drive is not None:
            # create configs home
            ConfUtil.create_drive(drive_name)
            # create remote
            if drive.open(**kwargs):
                drive.close(**kwargs)
            else:
                # don't create home if auth fails
                ConfUtil.remove_drive(drive_name)
        else:
            StdOut.display(msg=message.get(message.MISSING_IMPLEMENTATION, drive=drive_name))

class RemoteRm(FogCommand):

    def valid(self):
        # check argument size
        if len(self._args) != 1:
            StdOut.display(msg=message.get(message.INVALID_ARGS))
            return False

        drive_name = self._args[0]
        # reject invalid drive
        if not ConfUtil.valid_drive(drive_name):
            StdOut.display(msg=message.get(message.INVALID_DRIVE_NAME, drive=drive_name))
            return False

        # check if remote exists
        if not ConfUtil.exists_drive(drive_name):
            StdOut.display(msg=message.get(message.MISSING_REMOTE, drive=drive_name))
            return False

        return True

    def execute(self, **kwargs):
        drive_name = self._args[0]
        ConfUtil.remove_drive(drive_name)

class Pull(FogCommand):
    @staticmethod
    def name():
        return 'pull'

    def valid(self):
        if len(self._args) != 1 and len(self._args) != 2:
            return False

        # make sure drive has been checked out
        checkout = ConfUtil.get_checkout()
        if not checkout:
            StdOut.display(msg=message.get(message.MISSING_CHECKOUT))
            return False

        # make sure drive is tracked
        if not ConfUtil.exists_drive(checkout):
            StdOut.display(msg=message.get(message.MISSING_REMOTE, drive=checkout))
            return False

        return True

    def execute(self, **kwargs):
        # validate inputs
        src = self._args[0]
        dst = src
        if len(self._args) == 2:
            dst = self._args[1]

        drive = device.get_active_drive()
        # open connection
        if drive.open():
            # pull
            drive.download(src=src, dst=dst)
            #close
            drive.close()

class Invalid(FogCommand):

    def execute(self, **kwargs):
        StdOut.display(msg=message.get(message.INVALID_ARGS))

class Help(FogCommand):
    @staticmethod
    def name():
        return 'help'

    def execute(self, **kwargs):
        msgs = [
            'usage: fog <command> [<args>]',
            'Commonly used fog commands are:',
            'branch         List available drives',
            'checkout       Checkout a drive',
            'init           Create an empty fog directory or reinitialize an existing one',
            'pull           Download files from remote drive',
            'push           Upload a file to remote drive',
            'remote         List all the drives being tracked',
            'remote add     Track a remote drive',
            'remote rm      Un-track a remote drive',
            'rm             Move a file on remote drive to trash'
        ]

        for idx in range(0, 2, 1):
            StdOut.display(msg=msgs[idx])

        for idx in range(2, len(msgs), 1):
            StdOut.display(template=StdOut.IND2, msg=msgs[idx])
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-
import shutil

from flask_script import Command, Option, Server

import flask

import rdflib
from nanopub import Nanopublication
import tempfile

from whyis.namespace import np
from whyis.blueprint.nanopub.nanopub_utils import load_nanopub_graph

class LoadNanopub(Command):
    '''Add a nanopublication to the knowledge graph.'''

    _TEMP_STORE_DEFAULT = "Sleepycat"

    def get_options(self):
        return [
            Option('-i', '--input', dest='input_file', required=True, help='Path to file containing nanopub', type=str),
            Option('-f', '--format', dest='file_format', default='trig', help='File format (default: trig; also turtle, json-ld, xml, nquads, nt, rdfa)', type=str),
            Option('-r', '--revises', dest='was_revision_of', help="URI of nanopublication that this is a revision of", type=str),
            Option("--temp-store", dest="temp_store", type=str, default=self._TEMP_STORE_DEFAULT, help="backing store type to use for temporary graphs; deprecated")
        ]

    def run(self, input_file, file_format, temp_store=_TEMP_STORE_DEFAULT, was_revision_of=None):
        flask.current_app.managed = True
        if was_revision_of is not None:
            wasRevisionOf = set(flask.current_app.db.objects(predicate=np.hasAssertion,
                                                             subject=rdflib.URIRef(was_revision_of)))
            if len(wasRevisionOf) == 0:
                print("Could not find active nanopublication to revise:", was_revision_of)
                return
            was_revision_of = wasRevisionOf
        g = rdflib.ConjunctiveGraph(identifier=rdflib.BNode().skolemize(), store=temp_store)
        if temp_store == "Sleepycat":
            g_store_tempdir = tempfile.mkdtemp()
            g.store.open(g_store_tempdir, True)
        else:
            g_store_tempdir = None
        # g = rdflib.ConjunctiveGraph(identifier=rdflib.BNode().skolemize())

        try:
            g1 = load_nanopub_graph(location=input_file, format=file_format, store=g.store)

            try:
                nanopubs = []
                for npub in flask.current_app.nanopub_manager.prepare(g):
                    if was_revision_of is not None:
                        for r in was_revision_of:
                            print("Marking as revision of", r)
                            npub.pubinfo.add((npub.assertion.identifier, flask.current_app.NS.prov.wasRevisionOf, r))
                    print('Prepared', npub.identifier)
                    nanopubs.append(npub)
                flask.current_app.nanopub_manager.publish(*nanopubs)
                print("Published", npub.identifier)
            finally:
                if nanopub_prepare_graph_store_tempdir is not None:
                    shutil.rmtree(nanopub_prepare_graph_store_tempdir)
        finally:
            if g_store_tempdir is not None:
                shutil.rmtree(g_store_tempdir)
<commit_msg>Removed defunct use of a temp dir<commit_after># -*- coding:utf-8 -*-
import shutil

from flask_script import Command, Option, Server

import flask

import rdflib
from nanopub import Nanopublication
import tempfile

from whyis.namespace import np
from whyis.blueprint.nanopub.nanopub_utils import load_nanopub_graph

class LoadNanopub(Command):
    '''Add a nanopublication to the knowledge graph.'''

    _TEMP_STORE_DEFAULT = "Sleepycat"

    def get_options(self):
        return [
            Option('-i', '--input', dest='input_file', required=True, help='Path to file containing nanopub', type=str),
            Option('-f', '--format', dest='file_format', default='trig', help='File format (default: trig; also turtle, json-ld, xml, nquads, nt, rdfa)', type=str),
            Option('-r', '--revises', dest='was_revision_of', help="URI of nanopublication that this is a revision of", type=str),
            Option("--temp-store", dest="temp_store", type=str, default=self._TEMP_STORE_DEFAULT, help="backing store type to use for temporary graphs; deprecated")
        ]

    def run(self, input_file, file_format, temp_store=_TEMP_STORE_DEFAULT, was_revision_of=None):
        flask.current_app.managed = True
        if was_revision_of is not None:
            wasRevisionOf = set(flask.current_app.db.objects(predicate=np.hasAssertion,
                                                             subject=rdflib.URIRef(was_revision_of)))
            if len(wasRevisionOf) == 0:
                print("Could not find active nanopublication to revise:", was_revision_of)
                return
            was_revision_of = wasRevisionOf
        g = rdflib.ConjunctiveGraph(identifier=rdflib.BNode().skolemize(), store=temp_store)
        if temp_store == "Sleepycat":
            g_store_tempdir = tempfile.mkdtemp()
            g.store.open(g_store_tempdir, True)
        else:
            g_store_tempdir = None
        # g = rdflib.ConjunctiveGraph(identifier=rdflib.BNode().skolemize())

        try:
            g1 = load_nanopub_graph(location=input_file, format=file_format, store=g.store)

            nanopubs = []
            for npub in flask.current_app.nanopub_manager.prepare(g):
                if was_revision_of is not None:
                    for r in was_revision_of:
                        print("Marking as revision of", r)
                        npub.pubinfo.add((npub.assertion.identifier, flask.current_app.NS.prov.wasRevisionOf, r))
                print('Prepared', npub.identifier)
                nanopubs.append(npub)
            flask.current_app.nanopub_manager.publish(*nanopubs)
            print("Published", npub.identifier)
        finally:
            if g_store_tempdir is not None:
                shutil.rmtree(g_store_tempdir)
<|endoftext|>


<commit_before>from abc import ABCMeta, abstractmethod
from os import SEEK_SET

import six

from whylog.log_reader.const import BufsizeConsts

@six.add_metaclass(ABCMeta)
class AbstractSearcher(object):
    @abstractmethod
    def search(self, search_data):
        """
        transfer investigation to searcher
        """
        pass

class IndexSearcher(AbstractSearcher):
    pass

class DatabaseSearcher(AbstractSearcher):
    pass

class BacktrackSearcher(AbstractSearcher):
    def __init__(self, file_path):
        self._file_path = file_path

    def search(self, search_data):
        pass

    @classmethod
    def _decrease_actual_offset_properly(cls, actual_offset, decreasion_string):
        return actual_offset - len(decreasion_string) - 1

    def _reverse_from_offset(self, offset, buf_size=BufsizeConsts.STANDARD_BUF_SIZE):
        """
        a generator that returns the lines of a file in reverse order
        beginning with the specified offset
        """
        with open(self._file_path) as fh:
            fh.seek(offset)
            total_size = remaining_size = fh.tell()
            reverse_offset = 0
            actual_offset = offset
            truncated = None
            while remaining_size > 0:
                reverse_offset = min(total_size, reverse_offset + buf_size)
                fh.seek(total_size - reverse_offset, SEEK_SET)
                buffer_ = fh.read(min(remaining_size, buf_size))
                lines = buffer_.split('\n')
                remaining_size -= buf_size
                if truncated is not None:
                    if buffer_[-1] is not '\n':
                        lines[-1] += truncated
                    else:
                        actual_offset = self._decrease_actual_offset_properly(actual_offset,
                                                                              truncated)
                        yield truncated, actual_offset
                truncated = lines[0]
                for line in reversed(lines[1:]):
                    if len(line):
                        actual_offset = self._decrease_actual_offset_properly(actual_offset, line)
                        yield line, actual_offset
            actual_offset = self._decrease_actual_offset_properly(actual_offset, truncated)
            yield truncated, actual_offset
<commit_msg>Yapf changes<commit_after>from abc import ABCMeta, abstractmethod
from os import SEEK_SET

import six

from whylog.log_reader.const import BufsizeConsts

@six.add_metaclass(ABCMeta)
class AbstractSearcher(object):
    @abstractmethod
    def search(self, search_data):
        """
        transfer investigation to searcher
        """
        pass

class IndexSearcher(AbstractSearcher):
    pass

class DatabaseSearcher(AbstractSearcher):
    pass

class BacktrackSearcher(AbstractSearcher):
    def __init__(self, file_path):
        self._file_path = file_path

    def search(self, search_data):
        pass

    @classmethod
    def _decrease_actual_offset_properly(cls, actual_offset, decreasion_string):
        return actual_offset - len(decreasion_string) - 1

    def _reverse_from_offset(self, offset, buf_size=BufsizeConsts.STANDARD_BUF_SIZE):
        """
        a generator that returns the lines of a file in reverse order
        beginning with the specified offset
        """
        with open(self._file_path) as fh:
            fh.seek(offset)
            total_size = remaining_size = fh.tell()
            reverse_offset = 0
            actual_offset = offset
            truncated = None
            while remaining_size > 0:
                reverse_offset = min(total_size, reverse_offset + buf_size)
                fh.seek(total_size - reverse_offset, SEEK_SET)
                buffer_ = fh.read(min(remaining_size, buf_size))
                lines = buffer_.split('\n')
                remaining_size -= buf_size
                if truncated is not None:
                    if buffer_[-1] is not '\n':
                        lines[-1] += truncated
                    else:
                        actual_offset = self._decrease_actual_offset_properly(
                            actual_offset, truncated
                        )
                        yield truncated, actual_offset
                truncated = lines[0]
                for line in reversed(lines[1:]):
                    if len(line):
                        actual_offset = self._decrease_actual_offset_properly(actual_offset, line)
                        yield line, actual_offset
            actual_offset = self._decrease_actual_offset_properly(actual_offset, truncated)
            yield truncated, actual_offset
<|endoftext|>


<commit_before>import string
import re
import copy
import urlparse
from threading import Thread
from StringIO import StringIO
import requests
requests.adapters.DEFAULT_RETRIES = 20

from errors import error

forwarding_timeout = 15
default_headers = {}
required_forwarding_headers = [
    "X-Forward-ID",
    "X-Forward-To",
    "X-Forward-Query-Params",
    "X-Forward-Method",
    "X-Forward-Referer",
    "X-Forward-Errors-To"
]

def headers(header_dict):
    return dict(default_headers.items() + header_dict.items())

class Unforwardable(Exception): pass
class MissingHeaders(Exception): pass
class ResponseError(Exception): pass

def h_list(header):
    """ Convert a comma-separated HTTP header into a Python list. """
    obj = re.split(r",\s*", header)
    new_obj = []
    for item in obj:
        new_obj.append(string.replace(str(item), "%23", ","))
    return new_obj

def list_header(obj):
    """ Convert a Python list into a comma-separated HTTP header. """
    if obj.__class__ not in (list, tuple):
        return str(obj)
    new_obj = []
    for item in obj:
        new_obj.append(string.replace(str(item), ",", "%23"))
    return ", ".join(new_obj)

def encode_headers(out_headers):
    """ Convert a dictionary with Python lists into comma-separated HTTP header values. """
    headers = {}
    for header, value in out_headers.iteritems():
        if value.__class__ in (list, tuple):
            headers[header] = list_header(value)
        else:
            headers[header] = value

        if len(headers[header]) is 0:
            del headers[header]
    return headers

def dispatch_forwarding_request(iri=None, referer="", cookies={}, body="", b_headers={}):
    """ Dispatch a request to the next server in the forwarding list. """
    url = b_headers["X-Forward-To"].pop(0)
    query_params = b_headers["X-Forward-Query-Params"].pop(0)
    method = b_headers["X-Forward-Method"].pop(0).upper()

    scheme, netloc, path, params, old_query, fragment = urlparse.urlparse(url)
    full_url = urlparse.urlunparse((
        scheme,
        netloc,
        iri.manifestation_str(),
        params,
        query_params,
        ""
    ))

    b_headers["X-Forward-Referer"] = referer
    b_headers["Content-Type"] = iri.mime_type()
    full_url = string.replace(full_url, "#", "%23")

    try:
        resp = requests.request(method,
                                full_url,
                                headers=encode_headers(b_headers),
                                cookies=cookies,
                                allow_redirects=True,
                                timeout=forwarding_timeout,
                                data=StringIO(body.decode("utf-8")))
        if resp.status_code not in (requests.codes.accepted,
                                    requests.codes.created,
                                    requests.codes.ok,
                                    requests.codes.no_content):
            raise ResponseError(resp.status_code, resp.content)
    except ResponseError as e:
        error_url = b_headers.pop("X-Forward-Errors-To")[0]
        b_headers["X-Forward-Error-Condition"] = "External"
        b_headers["X-Forward-Error-Code"] = str(e.args[0])
        b_headers["X-Forward-Error-Message"] = str(e.args[1])
        requests.post(error_url,
                      headers=encode_headers(b_headers),
                      cookies=cookies,
                      allow_redirects=True,
                      timeout=forwarding_timeout,
                      data="")
    except Exception as e:
        # dispatch an error message
        error_url = b_headers.pop("X-Forward-Errors-To")[0]
        b_headers["X-Forward-Error-Condition"] = "Internal"
        b_headers["X-Forward-Error-Message"] = str(e)
        requests.post(error_url,
                      headers=encode_headers(b_headers),
                      cookies=cookies,
                      allow_redirects=True,
                      timeout=forwarding_timeout,
                      data="")

def parse_headers(in_headers):
    missing_headers = []
    headers = {}
    for header in required_forwarding_headers:
        if header in in_headers:
            headers[header] = h_list(in_headers[header])
        else:
            missing_headers.append(header)
    # no headers, so we assume it isn't forwardable
    if len(missing_headers) is len(required_forwarding_headers):
        raise Unforwardable()
    elif len(missing_headers) is 0:
        return headers
    else:
        raise MissingHeaders(missing_headers)

def handle_forwarding(response_body, request, new_iri):
    """
    Seamlessly either hand off the response to the next step in the
    forwarding chain, if there is one, or just return it to the client.
    """

    try:
        # DO NOT BLOCK. Dispatch the forwarding request and move on.
        t = Thread(target=dispatch_forwarding_request,
                   kwargs={
                       'referer': request.url,
                       'iri': new_iri,
                       'cookies': request.cookies,
                       'body': response_body,
                       'b_headers': parse_headers(request.headers)
                   })
        t.start()
        resp = ("",
                202,
                headers({
                    "X-Canonical-IRI": new_iri.manifestation_str()
                }))
    except Unforwardable as e:
        resp = (response_body,
                200,
                headers({
                    "X-Canonical-IRI": new_iri.manifestation_str()
                }))
    except MissingHeaders as e:
        missing_headers = e.args[0]
        if len(missing_headers) is 1:
            err_str = "Missing header %s" % missing_headers[0]
        else:
            header_tail_str = "and %s" % missing_headers.pop()
            err_str = "Missing headers %s %s" % (", ".join(missing_headers), header_tail_str)
        resp = error(400, err_str)
    return resp
<commit_msg>Get full stack trace on exception<commit_after>import sys
import traceback
import string
import re
import copy
import urlparse
from threading import Thread
from StringIO import StringIO
import requests
requests.adapters.DEFAULT_RETRIES = 20

from errors import error

forwarding_timeout = 15
default_headers = {}
required_forwarding_headers = [
    "X-Forward-ID",
    "X-Forward-To",
    "X-Forward-Query-Params",
    "X-Forward-Method",
    "X-Forward-Referer",
    "X-Forward-Errors-To"
]

def headers(header_dict):
    return dict(default_headers.items() + header_dict.items())

class Unforwardable(Exception): pass
class MissingHeaders(Exception): pass
class ResponseError(Exception): pass

def h_list(header):
    """ Convert a comma-separated HTTP header into a Python list. """
    obj = re.split(r",\s*", header)
    new_obj = []
    for item in obj:
        new_obj.append(string.replace(str(item), "%23", ","))
    return new_obj

def list_header(obj):
    """ Convert a Python list into a comma-separated HTTP header. """
    if obj.__class__ not in (list, tuple):
        return str(obj)
    new_obj = []
    for item in obj:
        new_obj.append(string.replace(str(item), ",", "%23"))
    return ", ".join(new_obj)

def encode_headers(out_headers):
    """ Convert a dictionary with Python lists into comma-separated HTTP header values. """
    headers = {}
    for header, value in out_headers.iteritems():
        if value.__class__ in (list, tuple):
            headers[header] = list_header(value)
        else:
            headers[header] = value

        if len(headers[header]) is 0:
            del headers[header]
    return headers

def dispatch_forwarding_request(iri=None, referer="", cookies={}, body="", b_headers={}):
    """ Dispatch a request to the next server in the forwarding list. """
    url = b_headers["X-Forward-To"].pop(0)
    query_params = b_headers["X-Forward-Query-Params"].pop(0)
    method = b_headers["X-Forward-Method"].pop(0).upper()

    scheme, netloc, path, params, old_query, fragment = urlparse.urlparse(url)
    full_url = urlparse.urlunparse((
        scheme,
        netloc,
        iri.manifestation_str(),
        params,
        query_params,
        ""
    ))

    b_headers["X-Forward-Referer"] = referer
    b_headers["Content-Type"] = iri.mime_type()
    full_url = string.replace(full_url, "#", "%23")

    try:
        resp = requests.request(method,
                                full_url,
                                headers=encode_headers(b_headers),
                                cookies=cookies,
                                allow_redirects=True,
                                timeout=forwarding_timeout,
                                data=body)
        if resp.status_code not in (requests.codes.accepted,
                                    requests.codes.created,
                                    requests.codes.ok,
                                    requests.codes.no_content):
            raise ResponseError(resp.status_code, resp.content)
    except ResponseError as e:
        error_url = b_headers.pop("X-Forward-Errors-To")[0]
        b_headers["X-Forward-Error-Condition"] = "External"
        b_headers["X-Forward-Error-Code"] = str(e.args[0])
        b_headers["X-Forward-Error-Message"] = str(e.args[1])
        requests.post(error_url,
                      headers=encode_headers(b_headers),
                      cookies=cookies,
                      allow_redirects=True,
                      timeout=forwarding_timeout,
                      data="")
    except Exception as e:
        # dispatch an error message
        msg = "\n".join(str(e), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1],
                sys.exc_info()[2]))
        error_url = b_headers.pop("X-Forward-Errors-To")[0]
        b_headers["X-Forward-Error-Condition"] = "Internal"
        b_headers["X-Forward-Error-Message"] = msg
        requests.post(error_url,
                      headers=encode_headers(b_headers),
                      cookies=cookies,
                      allow_redirects=True,
                      timeout=forwarding_timeout,
                      data="")

def parse_headers(in_headers):
    missing_headers = []
    headers = {}
    for header in required_forwarding_headers:
        if header in in_headers:
            headers[header] = h_list(in_headers[header])
        else:
            missing_headers.append(header)
    # no headers, so we assume it isn't forwardable
    if len(missing_headers) is len(required_forwarding_headers):
        raise Unforwardable()
    elif len(missing_headers) is 0:
        return headers
    else:
        raise MissingHeaders(missing_headers)

def handle_forwarding(response_body, request, new_iri):
    """
    Seamlessly either hand off the response to the next step in the
    forwarding chain, if there is one, or just return it to the client.
    """

    try:
        # DO NOT BLOCK. Dispatch the forwarding request and move on.
        t = Thread(target=dispatch_forwarding_request,
                   kwargs={
                       'referer': request.url,
                       'iri': new_iri,
                       'cookies': request.cookies,
                       'body': response_body,
                       'b_headers': parse_headers(request.headers)
                   })
        t.start()
        resp = ("",
                202,
                headers({
                    "X-Canonical-IRI": new_iri.manifestation_str()
                }))
    except Unforwardable as e:
        resp = (response_body,
                200,
                headers({
                    "X-Canonical-IRI": new_iri.manifestation_str()
                }))
    except MissingHeaders as e:
        missing_headers = e.args[0]
        if len(missing_headers) is 1:
            err_str = "Missing header %s" % missing_headers[0]
        else:
            header_tail_str = "and %s" % missing_headers.pop()
            err_str = "Missing headers %s %s" % (", ".join(missing_headers), header_tail_str)
        resp = error(400, err_str)
    return resp
<|endoftext|>


<commit_before>import tensorflow as tf
import random

class DataArg(object):
    def __init__(self):
        """ the image should be float32 type [0, 1]

        arg_dict["rbright_max"] = 0.2# [0,1]
        arg_dict["rcontrast_lower"] = 0.5 #[0,1]
        arg_dict["rcontrast_upper"] = 1.5 #[1,2]
        arg_dict["rhue_max"] = 0.2 #[0, 0.5]
        arg_dict["rsat_lower"] = 0.5 #[0,1]
        arg_dict["rsat_upper"] = 1.5 #[1,2]
        arg_dict["image_whiten"] = False #[False or True]
        arg_dict["rflip_updown"] = False #[False or True]
        arg_dict["rflip_leftright"] = False #[False or True]
        arg_dict["center_crop_frac"] = 0.2 #[0,1]

        arg_dict["rcrop_size"] = [256,256]
                    #random crop to size [crop_height, crop_width, crop_channel]

        arg_dict["ccrop_size"] = [256,256]
                    # center crop to size [crop_height, crop_width, crop_channel]

        """
        pass

    def __call__(self, data, arg_dict):
        seed = self.get_random_seed()
        if not isinstance(data, list):
            data = self.arg_single(data, arg_dict, seed, False)
        else:
            for i in range(len(data)):
                data[i] = self.arg_single(data[i], arg_dict[i], seed, True)
            data = self.arg_list(data, arg_dict[0], seed)

        return data

    def get_random_seed(self):
        return random.randint(0, 20000)

    def get_random_bboffset(self, wmax, hmax, seed):
        random.seed(seed)
        woffset = random.randint(0, wmax)
        hoffset = random.randint(0, hmax)
        return woffset, hoffset

    def arg_single(self,data, arg_dict, seed, is_list):
        """
            if seed == -1, the seed won't be set. For single data purpose
            if seed != -1, it will perform the same random for all
                    the data
        """

        #if seed != -1:
        #   tf.set_random_seed(seed)

        if "image_whiten" in arg_dict and arg_dict["image_whiten"]:
            data = tf.image.per_image_standardization(data)

        if "center_crop_frac" in arg_dict:
            data = tf.image.central_crop(data, arg_dict["center_crop_frac"])

        if "rbright_max" in arg_dict:
            data = tf.image.random_brightness(data,
                    arg_dict["rbright_max"],
                    seed = seed)

        if "rcontrast_lower_upper" in arg_dict:
            data = tf.image.random_contrast(data,
                    arg_dict["rcontrast_lower_upper"][0],
                    arg_dict["rcontrast_lower_upper"][1],
                    seed = seed)

        if "rhue_max" in arg_dict:
            data = tf.image.random_hue(data,
                    arg_dict["rhue_max"],
                    seed = seed)

        if "rsat_lower_upper" in arg_dict:
            data = tf.image.random_saturation(data,
                    arg_dict["rsat_lower_upper"][0],
                    arg_dict["rsat_lower_upper"][1],
                    seed = seed)

        if "ccrop_size" in arg_dict:
            i_height, i_width, i_cha = data.get_shape().as_list()
            ccrop_size = arg_dict["ccrop_size"]
            offset_height = int((i_height - ccrop_size[0])/2)
            offset_width = int((i_width - ccrop_size[1])/2)
            data = tf.image.crop_to_bounding_box(data,
                    offset_height, offset_width, ccrop_size[0], ccrop_size[1])

        if not is_list:
            if "rflip_updown" in arg_dict and arg_dict["rflip_updown"]:
                data = tf.image.random_flip_up_down(data, seed = seed)

            if "rflip_leftright" in arg_dict and arg_dict["rflip_leftright"]:
                data = tf.image.random_flip_left_right(data, seed)

            if "rcrop_size" in arg_dict:
                rcrop_size = arg_dict["rcrop_size"]
                data = tf.random_crop(data, rcrop_size, seed = seed)

        return data

    def arg_list(self, data, arg_dict, seed):
        assert(isinstance(data, list))

        if "rflip_updown" in arg_dict and arg_dict["rflip_updown"]:
            rflip_ud_op = tf.random_uniform([], minval = 0,
                        maxval = 2, dtype = tf.int32, seed = seed)
            mirror = tf.less(tf.pack([rflip_ud_op, 2, 2]), 1)
            for i in range(len(data)):
                data[i] = tf.reverse(data[i], mirror)

        if "rflip_leftright" in arg_dict and arg_dict["rflip_leftright"]:
            rflip_lr_op = tf.random_uniform([], minval = 0,
                        maxval = 2, dtype = tf.int32, seed = seed)
            mirror = tf.less(tf.pack([2, rflip_lr_op, 2]), 1)
            for i in range(len(data)):
                data[i] = tf.reverse(data[i], mirror)

        if "rcrop_size" in arg_dict:
            i_height, i_width, i_cha = data[0].get_shape().as_list()
            rcrop_size = arg_dict["rcrop_size"]
            offset_height_max = i_height - rcrop_size[0]
            offset_width_max = i_width - rcrop_size[1]

            if offset_height_max == 0 and offset_width_max == 0:
                pass
            else:
                r_weight = tf.random_uniform([],
                            minval = 0, maxval = offset_height_max,
                            dtype=tf.int32)

                r_width = tf.random_uniform([],
                            minval = 0, maxval = offset_width_max,
                            dtype=tf.int32)

                for i in range(len(data)):
                    data[i] = tf.image.crop_to_bounding_box(data[i],
                            r_weight, r_width, rcrop_size[0], rcrop_size[1])

        return data

            #if rcrop_size[2] == 1:
            #   rcrop_size[2] = 3
            #   data = tf.tile(data, [1,1,3])
            #   data = tf.random_crop(data,
            #       rcrop_size, seed = 0)
            #   data = data[:,:,1]
            #   data = tf.expand_dims(data, 2)
            #else:
            #   data = tf.random_crop(data,
            #       rcrop_size, seed = 0)
            #data_w = data.get_shape().as_list()[0]
            #data_h = data.get_shape().as_list()[1]
            #woffset, hoffset = self.get_random_bboffset(
            #           data_w - rcrop_size[0],
            #           data_h - rcrop_size[1], seed)

            #data = tf.image.crop_to_bounding_box(data,
            #           woffset, hoffset, rcrop_size[0], rcrop_size[1])

        #arg_dict.image_whiten = False [False or True]
        #arg_dict.center_crop_frac = 0.5 [0,1]

        #arg_dict.rbright_max = 0.2 [0,1]
        #arg_dict.rcontrast_lower_upper = [0.5, 1.5] [0,1] [1,2]
        #arg_dict.rhue_max = 0.2 [0, 0.5]
        #arg_dict.rsat_lower_upper = [0.5, 1.5] [0,1] [1,2]
        #arg_dict.rflip_updown = False [False or True]
        #arg_dict.rflip_leftright = False [False or True]
        #arg_dict.rcrop_size = [256,256,3] [crop_height, crop_width, crop_channel]

        #tf.image.random_brightness(image, max_delta, seed=None)
        #tf.image.random_contrast(image, lower, upper, seed=None)
        #tf.image.random_hue(image, max_delta, seed=None)
        #tf.image.random_saturation(image, lower, upper, seed=None)
        #tf.image.random_flip_up_down(image, seed=None)
        #tf.image.random_flip_left_right(image, seed=None)
        #tf.random_crop(value, size, seed=None, name=None)

<commit_msg>change data args<commit_after>import tensorflow as tf
import random

class DataArg(object):
    def __init__(self):
        """ the image should be float32 type [0, 1]

        arg_dict["rbright_max"] = 0.2# [0,1]
        arg_dict["rcontrast_lower"] = 0.5 #[0,1]
        arg_dict["rcontrast_upper"] = 1.5 #[1,2]
        arg_dict["rhue_max"] = 0.2 #[0, 0.5]
        arg_dict["rsat_lower"] = 0.5 #[0,1]
        arg_dict["rsat_upper"] = 1.5 #[1,2]
        arg_dict["image_whiten"] = False #[False or True]
        arg_dict["rflip_updown"] = False #[False or True]
        arg_dict["rflip_leftright"] = False #[False or True]
        arg_dict["center_crop_frac"] = 0.2 #[0,1]

        arg_dict["rcrop_size"] = [256,256]
                    #random crop to size [crop_height, crop_width, crop_channel]

        arg_dict["ccrop_size"] = [256,256]
                    # center crop to size [crop_height, crop_width, crop_channel]

        """
        pass

    def __call__(self, data, arg_dict):
        seed = self.get_random_seed()
        if not isinstance(data, list):
            data = self.arg_single(data, arg_dict, seed, False)
        else:
            for i in range(len(data)):
                data[i] = self.arg_single(data[i], arg_dict[i], seed, True)
            data = self.arg_list(data, arg_dict, seed)

        return data

    def get_random_seed(self):
        return random.randint(0, 20000)

    def get_random_bboffset(self, wmax, hmax, seed):
        random.seed(seed)
        woffset = random.randint(0, wmax)
        hoffset = random.randint(0, hmax)
        return woffset, hoffset

    def arg_single(self,data, arg_dict, seed, is_list):
        """
            if seed == -1, the seed won't be set. For single data purpose
            if seed != -1, it will perform the same random for all
                    the data
        """

        #if seed != -1:
        #   tf.set_random_seed(seed)

        if "image_whiten" in arg_dict and arg_dict["image_whiten"]:
            data = tf.image.per_image_standardization(data)

        if "center_crop_frac" in arg_dict:
            data = tf.image.central_crop(data, arg_dict["center_crop_frac"])

        if "rbright_max" in arg_dict:
            data = tf.image.random_brightness(data,
                    arg_dict["rbright_max"],
                    seed = seed)

        if "rcontrast_lower_upper" in arg_dict:
            data = tf.image.random_contrast(data,
                    arg_dict["rcontrast_lower_upper"][0],
                    arg_dict["rcontrast_lower_upper"][1],
                    seed = seed)

        if "rhue_max" in arg_dict:
            data = tf.image.random_hue(data,
                    arg_dict["rhue_max"],
                    seed = seed)

        if "rsat_lower_upper" in arg_dict:
            data = tf.image.random_saturation(data,
                    arg_dict["rsat_lower_upper"][0],
                    arg_dict["rsat_lower_upper"][1],
                    seed = seed)

        if "ccrop_size" in arg_dict:
            i_height, i_width, i_cha = data.get_shape().as_list()
            ccrop_size = arg_dict["ccrop_size"]
            offset_height = int((i_height - ccrop_size[0])/2)
            offset_width = int((i_width - ccrop_size[1])/2)
            data = tf.image.crop_to_bounding_box(data,
                    offset_height, offset_width, ccrop_size[0], ccrop_size[1])

        if not is_list:
            if "rflip_updown" in arg_dict and arg_dict["rflip_updown"]:
                data = tf.image.random_flip_up_down(data, seed = seed)

            if "rflip_leftright" in arg_dict and arg_dict["rflip_leftright"]:
                data = tf.image.random_flip_left_right(data, seed)

            if "rcrop_size" in arg_dict:
                rcrop_size = arg_dict["rcrop_size"]
                data = tf.random_crop(data, rcrop_size, seed = seed)

        return data

    def arg_list(self, data, arg_dict, seed):
        assert(isinstance(data, list))
        assert(len(data) == len(arg_dict))

        """for up down flip """
        rflip_ud_op = tf.random_uniform([], minval = 0,
                    maxval = 2, dtype = tf.int32, seed = seed)
        mirror = tf.less(tf.pack([rflip_ud_op, 2, 2]), 1)
        for i in range(len(data)):
            if "rflip_updown" in arg_dict[i] and arg_dict[i]["rflip_updown"]:
                data[i] = tf.reverse(data[i], mirror)

        """for left right flip """
        rflip_lr_op = tf.random_uniform([], minval = 0,
                    maxval = 2, dtype = tf.int32, seed = seed)
        mirror = tf.less(tf.pack([2, rflip_lr_op, 2]), 1)
        for i in range(len(data)):
            if "rflip_leftright" in arg_dict[i] and arg_dict[i]["rflip_leftright"]:
                data[i] = tf.reverse(data[i], mirror)

        """ for random crop """
        activate_rcrop = False
        for i in range(len(data)):
            if "rcrop_size" in arg_dict[i]:
                if not activate_rcrop:
                    activate_rcrop = True
                    i_height, i_width, i_cha = data[i].get_shape().as_list()
                    rcrop_size = arg_dict[i]["rcrop_size"]
                    offset_height_max = i_height - rcrop_size[0]
                    offset_width_max = i_width - rcrop_size[1]

                    if offset_height_max == 0 and offset_width_max == 0:
                        pass
                    else:
                        r_weight = tf.random_uniform([],
                                    minval = 0, maxval = offset_height_max,
                                    dtype=tf.int32)

                        r_width = tf.random_uniform([],
                                    minval = 0, maxval = offset_width_max,
                                    dtype=tf.int32)

                if offset_height_max == 0 and offset_width_max == 0:
                    pass
                else:
                    data[i] = tf.image.crop_to_bounding_box(data[i],
                            r_weight, r_width, rcrop_size[0], rcrop_size[1])

        return data

        #if "rflip_updown" in arg_dict and arg_dict["rflip_updown"]:
        #    rflip_ud_op = tf.random_uniform([], minval = 0,
        #                maxval = 2, dtype = tf.int32, seed = seed)
        #    mirror = tf.less(tf.pack([rflip_ud_op, 2, 2]), 1)
        #    for i in range(len(data)):
        #        data[i] = tf.reverse(data[i], mirror)

        #if "rflip_leftright" in arg_dict and arg_dict["rflip_leftright"]:
        #    rflip_lr_op = tf.random_uniform([], minval = 0,
        #                maxval = 2, dtype = tf.int32, seed = seed)
        #    mirror = tf.less(tf.pack([2, rflip_lr_op, 2]), 1)
        #    for i in range(len(data)):
        #        data[i] = tf.reverse(data[i], mirror)

        #if "rcrop_size" in arg_dict:
        #    i_height, i_width, i_cha = data[0].get_shape().as_list()
        #    rcrop_size = arg_dict["rcrop_size"]
        #    offset_height_max = i_height - rcrop_size[0]
        #    offset_width_max = i_width - rcrop_size[1]

        #    if offset_height_max == 0 and offset_width_max == 0:
        #        pass
        #    else:
        #        r_weight = tf.random_uniform([],
        #                    minval = 0, maxval = offset_height_max,
        #                    dtype=tf.int32)

        #        r_width = tf.random_uniform([],
        #                    minval = 0, maxval = offset_width_max,
        #                    dtype=tf.int32)

        #        for i in range(len(data)):
        #            data[i] = tf.image.crop_to_bounding_box(data[i],
        #                    r_weight, r_width, rcrop_size[0], rcrop_size[1])

        #return data

            #if rcrop_size[2] == 1:
            #   rcrop_size[2] = 3
            #   data = tf.tile(data, [1,1,3])
            #   data = tf.random_crop(data,
            #       rcrop_size, seed = 0)
            #   data = data[:,:,1]
            #   data = tf.expand_dims(data, 2)
            #else:
            #   data = tf.random_crop(data,
            #       rcrop_size, seed = 0)
            #data_w = data.get_shape().as_list()[0]
            #data_h = data.get_shape().as_list()[1]
            #woffset, hoffset = self.get_random_bboffset(
            #           data_w - rcrop_size[0],
            #           data_h - rcrop_size[1], seed)

            #data = tf.image.crop_to_bounding_box(data,
            #           woffset, hoffset, rcrop_size[0], rcrop_size[1])

        #arg_dict.image_whiten = False [False or True]
        #arg_dict.center_crop_frac = 0.5 [0,1]

        #arg_dict.rbright_max = 0.2 [0,1]
        #arg_dict.rcontrast_lower_upper = [0.5, 1.5] [0,1] [1,2]
        #arg_dict.rhue_max = 0.2 [0, 0.5]
        #arg_dict.rsat_lower_upper = [0.5, 1.5] [0,1] [1,2]
        #arg_dict.rflip_updown = False [False or True]
        #arg_dict.rflip_leftright = False [False or True]
        #arg_dict.rcrop_size = [256,256,3] [crop_height, crop_width, crop_channel]

        #tf.image.random_brightness(image, max_delta, seed=None)
        #tf.image.random_contrast(image, lower, upper, seed=None)
        #tf.image.random_hue(image, max_delta, seed=None)
        #tf.image.random_saturation(image, lower, upper, seed=None)
        #tf.image.random_flip_up_down(image, seed=None)
        #tf.image.random_flip_left_right(image, seed=None)
        #tf.random_crop(value, size, seed=None, name=None)

<|endoftext|>


<commit_before>import chardet
import json
import os
import re
import subprocess
import tempfile
from cStringIO import StringIO

import validator.testcases.javascript.traverser as traverser
from validator.constants import SPIDERMONKEY_INSTALLATION
from validator.contextgenerator import ContextGenerator
from validator.textfilter import *

JS_ESCAPE = re.compile("\\\\+[ux]", re.I)

def test_js_file(err, filename, data, line=0):
    "Tests a JS file by parsing and analyzing its tokens"

    if SPIDERMONKEY_INSTALLATION is None or \
       err.get_resource("SPIDERMONKEY") is None: # Default value is False
        return

    before_tier = None
    # Set the tier to 4 (Security Tests)
    if err is not None:
        before_tier = err.tier
        err.tier = 4

    # Get the AST tree for the JS code
    try:
        tree = _get_tree(filename,
                         data,
                         shell=(err and err.get_resource("SPIDERMONKEY")) or
                               SPIDERMONKEY_INSTALLATION,
                         errorbundle=err)

    except JSReflectException as exc:
        str_exc = str(exc).strip("'\"")
        if "SyntaxError" in str_exc:
            err.warning(("testcases_scripting",
                         "test_js_file",
                         "syntax_error"),
                         "JavaScript Syntax Error",
                         ["A syntax error in the JavaScript halted validation "
                          "of that file.",
                          "Message: %s" % str_exc[15:-1]],
                         filename=filename,
                         line=exc.line,
                         context=ContextGenerator(data))
        elif "InternalError: too much recursion" in str_exc:
            err.notice(("testcases_scripting",
                        "test_js_file",
                        "recursion_error"),
                       "JS too deeply nested for validation",
                       "A JS file was encountered that could not be valiated "
                       "due to limitations with Spidermonkey. It should be "
                       "manually inspected.",
                       filename=filename)
        else:
            err.warning(("testcases_scripting",
                         "test_js_file",
                         "retrieving_tree"),
                        "JS reflection error prevented validation",
                        ["An error in the JavaScript file prevented it from "
                         "being properly read by the Spidermonkey JS engine.",
                         str(exc)],
                        filename=filename)
            import sys
            etype, err, tb = sys.exc_info()
            raise exc, None, tb

        if before_tier:
            err.tier = before_tier
        return

    if tree is None:
        if before_tier:
            err.tier = before_tier
        return None

    context = ContextGenerator(data)
    if traverser.DEBUG:
        _do_test(err=err, filename=filename, line=line, context=context,
                 tree=tree)
    else:
        try:
            _do_test(err=err, filename=filename, line=line, context=context,
                     tree=tree)
        except:
            # We do this because the validator can still be damn unstable.
            pass

    _regex_tests(err, data, filename)

    # Reset the tier so we don't break the world
    if err is not None:
        err.tier = before_tier

def test_js_snippet(err, data, filename, line=0):
    "Process a JS snippet by passing it through to the file tester."

    # Wrap snippets in a function to prevent the parser from freaking out
    # when return statements exist without a corresponding function.
    data = "(function(){%s\n})()" % data

    test_js_file(err, filename, data, line)

def _do_test(err, filename, line, context, tree):
    t = traverser.Traverser(err, filename, line, context=context)
    t.run(tree)

def _regex_tests(err, data, filename):

    c = ContextGenerator(data)

    np_warning = "Network preferences may not be modified."

    errors = {"globalStorage\\[.*\\].password":
                  "Global Storage may not be used to store passwords.",
              "network\\.http": np_warning,
              "extensions\\.blocklist\\.url": np_warning,
              "extensions\\.blocklist\\.level": np_warning,
              "extensions\\.blocklist\\.interval": np_warning,
              "general\\.useragent": np_warning}

    for regex, message in errors.items():
        reg = re.compile(regex)
        match = reg.search(data)

        if match:
            line = c.get_line(match.start())
            err.warning(("testcases_scripting",
                         "regex_tests",
                         "compiled_error"),
                        "Potentially malicious JS",
                        message,
                        filename=filename,
                        line=line,
                        context=c)

class JSReflectException(Exception):
    "An exception to indicate that tokenization has failed"

    def __init__(self, value):
        self.value = value
        self.line = None

    def __str__(self):
        return repr(self.value)

    def line_num(self, line_num):
        "Set the line number and return self for chaining"
        self.line = int(line_num)
        return self

def strip_weird_chars(chardata, err=None, name=""):
    line_num = 1
    out_code = StringIO()
    has_warned_ctrlchar = False

    for line in chardata.split("\n"):

        charpos = 0
        for char in line:
            if is_standard_ascii(char):
                out_code.write(char)
            else:
                if not has_warned_ctrlchar and err is not None:
                    err.warning(("testcases_scripting",
                                 "_get_tree",
                                 "control_char_filter"),
                                "Invalid control character in JS file",
                                "An invalid character (ASCII 0-31, except CR "
                                "and LF) has been found in a JS file. These "
                                "are considered unsafe and should be removed.",
                                filename=name,
                                line=line_num,
                                column=charpos,
                                context=ContextGenerator(chardata))
                has_warned_ctrlchar = True

            charpos += 1

        out_code.write("\n")
        line_num += 1

    return out_code.getvalue()

def _get_tree(name, code, shell=SPIDERMONKEY_INSTALLATION, errorbundle=None):
    "Returns an AST tree of the JS passed in `code`."

    if not code:
        return None

    # Acceptable unicode characters still need to be stripped. Just remove the
    # slash: a character is necessary to prevent bad identifier errors
    code = JS_ESCAPE.sub("u", code)

    encoding = None
    try:
        code = unicode(code) # Make sure we can get a Unicode representation
        code = strip_weird_chars(code, errorbundle, name=name)
    except UnicodeDecodeError:
        # If it's not an easily decodeable encoding, detect it and decode that
        code = filter_ascii(code)

    data = json.dumps(code)
    data = """try{
        print(JSON.stringify(Reflect.parse(%s)));
    } catch(e) {
        print(JSON.stringify({
            "error":true,
            "error_message":e.toString(),
            "line_number":e.lineNumber
        }));
    }""" % data

    # Push the data to a temporary file
    temp = tempfile.NamedTemporaryFile(mode="w+", delete=True)
    temp.write(data)
    temp.flush() # This is very important

    cmd = [shell, "-f", temp.name]
    try:
        shell = subprocess.Popen(cmd,
    	                     shell=False,
    			     stderr=subprocess.PIPE,
    			     stdout=subprocess.PIPE)
    except OSError:
        raise OSError("Spidermonkey shell could not be run.")

    data, stderr = shell.communicate()
    if stderr: raise RuntimeError('Error calling %r: %s' % (cmd, stderr))

    # Closing the temp file will delete it.
    temp.close()

    try:
        data = unicode(data)
    except UnicodeDecodeError:
        data = unicode(filter_ascii(data))

    parsed = json.loads(data, strict=False)

    if "error" in parsed and parsed["error"]:
        if parsed["error_message"][:14] == "ReferenceError":
            raise RuntimeError("Spidermonkey version too old; "
                               "1.8pre+ required")
        else:
            raise JSReflectException(parsed["error_message"]).line_num(
                    parsed["line_number"])

    return parsed

<commit_msg>Update for createElement; bug 625690<commit_after>import chardet
import json
import os
import re
import subprocess
import tempfile
from cStringIO import StringIO

import validator.testcases.javascript.traverser as traverser
from validator.constants import SPIDERMONKEY_INSTALLATION
from validator.contextgenerator import ContextGenerator
from validator.textfilter import *

JS_ESCAPE = re.compile("\\\\+[ux]", re.I)

def test_js_file(err, filename, data, line=0):
    "Tests a JS file by parsing and analyzing its tokens"

    if SPIDERMONKEY_INSTALLATION is None or \
       err.get_resource("SPIDERMONKEY") is None: # Default value is False
        return

    before_tier = None
    # Set the tier to 4 (Security Tests)
    if err is not None:
        before_tier = err.tier
        err.tier = 4

    # Get the AST tree for the JS code
    try:
        tree = _get_tree(filename,
                         data,
                         shell=(err and err.get_resource("SPIDERMONKEY")) or
                               SPIDERMONKEY_INSTALLATION,
                         errorbundle=err)

    except JSReflectException as exc:
        str_exc = str(exc).strip("'\"")
        if "SyntaxError" in str_exc:
            err.warning(("testcases_scripting",
                         "test_js_file",
                         "syntax_error"),
                         "JavaScript Syntax Error",
                         ["A syntax error in the JavaScript halted validation "
                          "of that file.",
                          "Message: %s" % str_exc[15:-1]],
                         filename=filename,
                         line=exc.line,
                         context=ContextGenerator(data))
        elif "InternalError: too much recursion" in str_exc:
            err.notice(("testcases_scripting",
                        "test_js_file",
                        "recursion_error"),
                       "JS too deeply nested for validation",
                       "A JS file was encountered that could not be valiated "
                       "due to limitations with Spidermonkey. It should be "
                       "manually inspected.",
                       filename=filename)
        else:
            err.warning(("testcases_scripting",
                         "test_js_file",
                         "retrieving_tree"),
                        "JS reflection error prevented validation",
                        ["An error in the JavaScript file prevented it from "
                         "being properly read by the Spidermonkey JS engine.",
                         str(exc)],
                        filename=filename)
            import sys
            etype, err, tb = sys.exc_info()
            raise exc, None, tb

        if before_tier:
            err.tier = before_tier
        return

    if tree is None:
        if before_tier:
            err.tier = before_tier
        return None

    context = ContextGenerator(data)
    if traverser.DEBUG:
        _do_test(err=err, filename=filename, line=line, context=context,
                 tree=tree)
    else:
        try:
            _do_test(err=err, filename=filename, line=line, context=context,
                     tree=tree)
        except:
            # We do this because the validator can still be damn unstable.
            pass

    _regex_tests(err, data, filename)

    # Reset the tier so we don't break the world
    if err is not None:
        err.tier = before_tier

def test_js_snippet(err, data, filename, line=0):
    "Process a JS snippet by passing it through to the file tester."

    # Wrap snippets in a function to prevent the parser from freaking out
    # when return statements exist without a corresponding function.
    data = "(function(){%s\n})()" % data

    test_js_file(err, filename, data, line)

def _do_test(err, filename, line, context, tree):
    t = traverser.Traverser(err, filename, line, context=context)
    t.run(tree)

def _regex_tests(err, data, filename):

    c = ContextGenerator(data)

    np_warning = "Network preferences may not be modified."

    errors = {"globalStorage\\[.*\\].password":
                  "Global Storage may not be used to store passwords.",
              "network\\.http": np_warning,
              "extensions\\.blocklist\\.url": np_warning,
              "extensions\\.blocklist\\.level": np_warning,
              "extensions\\.blocklist\\.interval": np_warning,
              "general\\.useragent": np_warning,
              "createElement": "Markup elements may not be created dynamically"}

    for regex, message in errors.items():
        reg = re.compile(regex)
        match = reg.search(data)

        if match:
            line = c.get_line(match.start())
            err.warning(("testcases_scripting",
                         "regex_tests",
                         "compiled_error"),
                        "Potentially malicious JS",
                        message,
                        filename=filename,
                        line=line,
                        context=c)

class JSReflectException(Exception):
    "An exception to indicate that tokenization has failed"

    def __init__(self, value):
        self.value = value
        self.line = None

    def __str__(self):
        return repr(self.value)

    def line_num(self, line_num):
        "Set the line number and return self for chaining"
        self.line = int(line_num)
        return self

def strip_weird_chars(chardata, err=None, name=""):
    line_num = 1
    out_code = StringIO()
    has_warned_ctrlchar = False

    for line in chardata.split("\n"):

        charpos = 0
        for char in line:
            if is_standard_ascii(char):
                out_code.write(char)
            else:
                if not has_warned_ctrlchar and err is not None:
                    err.warning(("testcases_scripting",
                                 "_get_tree",
                                 "control_char_filter"),
                                "Invalid control character in JS file",
                                "An invalid character (ASCII 0-31, except CR "
                                "and LF) has been found in a JS file. These "
                                "are considered unsafe and should be removed.",
                                filename=name,
                                line=line_num,
                                column=charpos,
                                context=ContextGenerator(chardata))
                has_warned_ctrlchar = True

            charpos += 1

        out_code.write("\n")
        line_num += 1

    return out_code.getvalue()

def _get_tree(name, code, shell=SPIDERMONKEY_INSTALLATION, errorbundle=None):
    "Returns an AST tree of the JS passed in `code`."

    if not code:
        return None

    # Acceptable unicode characters still need to be stripped. Just remove the
    # slash: a character is necessary to prevent bad identifier errors
    code = JS_ESCAPE.sub("u", code)

    encoding = None
    try:
        code = unicode(code) # Make sure we can get a Unicode representation
        code = strip_weird_chars(code, errorbundle, name=name)
    except UnicodeDecodeError:
        # If it's not an easily decodeable encoding, detect it and decode that
        code = filter_ascii(code)

    data = json.dumps(code)
    data = """try{
        print(JSON.stringify(Reflect.parse(%s)));
    } catch(e) {
        print(JSON.stringify({
            "error":true,
            "error_message":e.toString(),
            "line_number":e.lineNumber
        }));
    }""" % data

    # Push the data to a temporary file
    temp = tempfile.NamedTemporaryFile(mode="w+", delete=True)
    temp.write(data)
    temp.flush() # This is very important

    cmd = [shell, "-f", temp.name]
    try:
        shell = subprocess.Popen(cmd,
    	                     shell=False,
    			     stderr=subprocess.PIPE,
    			     stdout=subprocess.PIPE)
    except OSError:
        raise OSError("Spidermonkey shell could not be run.")

    data, stderr = shell.communicate()
    if stderr: raise RuntimeError('Error calling %r: %s' % (cmd, stderr))

    # Closing the temp file will delete it.
    temp.close()

    try:
        data = unicode(data)
    except UnicodeDecodeError:
        data = unicode(filter_ascii(data))

    parsed = json.loads(data, strict=False)

    if "error" in parsed and parsed["error"]:
        if parsed["error_message"][:14] == "ReferenceError":
            raise RuntimeError("Spidermonkey version too old; "
                               "1.8pre+ required")
        else:
            raise JSReflectException(parsed["error_message"]).line_num(
                    parsed["line_number"])

    return parsed

<|endoftext|>


<commit_before>import importlib
import warnings
from types import ModuleType

from django.apps import AppConfig

from kolibri.plugins.registry import registered_plugins
from kolibri.utils import i18n

def _validate_settings_module(settings_module):
    if isinstance(settings_module, str):
        try:
            return importlib.import_module(settings_module)
        except ImportError:
            raise ValueError(
                "Invalid settings module path {path}".format(path=settings_module)
            )
    elif not isinstance(settings_module, ModuleType):
        raise TypeError(
            "Invalid argument for apply_settings - requires module or module path"
        )
    return settings_module

_tuple_settings = ("INSTALLED_APPS", "TEMPLATE_DIRS", "LOCALE_PATHS", "AUTHENTICATION_BACKENDS")

def _validate_module_setting(
    setting, existing_settings, setting_value, plugin_settings, module_path
):
    # Raise an error if overwriting an existing setting that cannot be simply appended
    # to.
    if setting in existing_settings and setting not in _tuple_settings:
        raise ValueError(
            "Plugin settings should not be used to override Kolibri default settings, use the Django settings module env var"
        )
    # Raise an error if a tuple setting is not an iterable.
    if setting in _tuple_settings and not isinstance(setting_value, (tuple, list)):
        raise ValueError("{setting} must be a tuple or a list".format(setting=setting))
    # Warn if this setting has already been modified by another plugin
    if setting in plugin_settings:
        warnings.warn(
            "Plugin {plugin_module} is modifying the {setting} setting, but this was already modified by: {plugins}".format(
                plugin_module=module_path,
                setting=setting,
                plugins=", ".join(plugin_settings[setting]),
            )
        )
        plugin_settings[setting].append(module_path)
    else:
        plugin_settings[setting] = [module_path]

def _set_setting_value(setting, setting_value, settings_module):
    if setting in _tuple_settings:
        setting_value = tuple(setting_value)
        original_value = tuple(getattr(settings_module, setting, tuple()))
        setattr(settings_module, setting, original_value + setting_value)
    else:
        setattr(settings_module, setting, setting_value)

def _process_module_settings(
    plugin_settings_module,
    plugin_settings,
    existing_settings,
    module_path,
    settings_module,
):
    # Settings setting inspired by how Django does it internally:
    # https://github.com/django/django/blob/stable/1.11.x/django/conf/__init__.py#L100
    for setting in dir(plugin_settings_module):
        # All Django settings are all caps, so only read these.
        if setting.isupper():
            setting_value = getattr(plugin_settings_module, setting)
            _validate_module_setting(
                setting, existing_settings, setting_value, plugin_settings, module_path
            )
            _set_setting_value(setting, setting_value, settings_module)

def _apply_base_settings(plugin_instance, settings_module):
    # Instead of just adding the module path to the settings
    # we instantiate an app config object for the plugin
    # and explicitly set its label to its module path.
    # This way, there is no way for a plugin to collide in its
    # label in the Django App Registry with kolibri core apps
    # or Kolibri core plugins.
    app_config = AppConfig.create(plugin_instance.module_path)
    app_config.label = plugin_instance.module_path
    # Register the plugin as an installed app
    _set_setting_value("INSTALLED_APPS", (app_config,), settings_module)
    # Add in the external plugins' locale paths. Our frontend messages depends
    # specifically on the value of LOCALE_PATHS to find its catalog files.
    if i18n.is_external_plugin(plugin_instance.module_path):
        _set_setting_value(
            "LOCALE_PATHS",
            (i18n.get_installed_app_locale_path(plugin_instance.module_path),),
            settings_module,
        )

def apply_settings(settings_module):
    from django.conf import settings

    if settings.configured:
        raise RuntimeError(
            "Attempted to apply settings from plugins after Django settings have been configured"
        )

    settings_module = _validate_settings_module(settings_module)

    # Keep track of which settings already exist to error if a plugin
    # tries to override them.
    # Do this here so that we can then warn but allow if plugin settings collide.
    existing_settings = dir(settings_module)

    # Keep track of which plugins have modified settings
    plugin_settings = {}

    for plugin_instance in registered_plugins:
        _apply_base_settings(plugin_instance, settings_module)
        plugin_settings_module = plugin_instance.settings_module
        if plugin_settings_module:
            _process_module_settings(
                plugin_settings_module,
                plugin_settings,
                existing_settings,
                plugin_instance.module_path,
                settings_module,
            )
<commit_msg>Running black on the python code<commit_after>import importlib
import warnings
from types import ModuleType

from django.apps import AppConfig

from kolibri.plugins.registry import registered_plugins
from kolibri.utils import i18n

def _validate_settings_module(settings_module):
    if isinstance(settings_module, str):
        try:
            return importlib.import_module(settings_module)
        except ImportError:
            raise ValueError(
                "Invalid settings module path {path}".format(path=settings_module)
            )
    elif not isinstance(settings_module, ModuleType):
        raise TypeError(
            "Invalid argument for apply_settings - requires module or module path"
        )
    return settings_module

_tuple_settings = (
    "INSTALLED_APPS",
    "TEMPLATE_DIRS",
    "LOCALE_PATHS",
    "AUTHENTICATION_BACKENDS",
)

def _validate_module_setting(
    setting, existing_settings, setting_value, plugin_settings, module_path
):
    # Raise an error if overwriting an existing setting that cannot be simply appended
    # to.
    if setting in existing_settings and setting not in _tuple_settings:
        raise ValueError(
            "Plugin settings should not be used to override Kolibri default settings, use the Django settings module env var"
        )
    # Raise an error if a tuple setting is not an iterable.
    if setting in _tuple_settings and not isinstance(setting_value, (tuple, list)):
        raise ValueError("{setting} must be a tuple or a list".format(setting=setting))
    # Warn if this setting has already been modified by another plugin
    if setting in plugin_settings:
        warnings.warn(
            "Plugin {plugin_module} is modifying the {setting} setting, but this was already modified by: {plugins}".format(
                plugin_module=module_path,
                setting=setting,
                plugins=", ".join(plugin_settings[setting]),
            )
        )
        plugin_settings[setting].append(module_path)
    else:
        plugin_settings[setting] = [module_path]

def _set_setting_value(setting, setting_value, settings_module):
    if setting in _tuple_settings:
        setting_value = tuple(setting_value)
        original_value = tuple(getattr(settings_module, setting, tuple()))
        setattr(settings_module, setting, original_value + setting_value)
    else:
        setattr(settings_module, setting, setting_value)

def _process_module_settings(
    plugin_settings_module,
    plugin_settings,
    existing_settings,
    module_path,
    settings_module,
):
    # Settings setting inspired by how Django does it internally:
    # https://github.com/django/django/blob/stable/1.11.x/django/conf/__init__.py#L100
    for setting in dir(plugin_settings_module):
        # All Django settings are all caps, so only read these.
        if setting.isupper():
            setting_value = getattr(plugin_settings_module, setting)
            _validate_module_setting(
                setting, existing_settings, setting_value, plugin_settings, module_path
            )
            _set_setting_value(setting, setting_value, settings_module)

def _apply_base_settings(plugin_instance, settings_module):
    # Instead of just adding the module path to the settings
    # we instantiate an app config object for the plugin
    # and explicitly set its label to its module path.
    # This way, there is no way for a plugin to collide in its
    # label in the Django App Registry with kolibri core apps
    # or Kolibri core plugins.
    app_config = AppConfig.create(plugin_instance.module_path)
    app_config.label = plugin_instance.module_path
    # Register the plugin as an installed app
    _set_setting_value("INSTALLED_APPS", (app_config,), settings_module)
    # Add in the external plugins' locale paths. Our frontend messages depends
    # specifically on the value of LOCALE_PATHS to find its catalog files.
    if i18n.is_external_plugin(plugin_instance.module_path):
        _set_setting_value(
            "LOCALE_PATHS",
            (i18n.get_installed_app_locale_path(plugin_instance.module_path),),
            settings_module,
        )

def apply_settings(settings_module):
    from django.conf import settings

    if settings.configured:
        raise RuntimeError(
            "Attempted to apply settings from plugins after Django settings have been configured"
        )

    settings_module = _validate_settings_module(settings_module)

    # Keep track of which settings already exist to error if a plugin
    # tries to override them.
    # Do this here so that we can then warn but allow if plugin settings collide.
    existing_settings = dir(settings_module)

    # Keep track of which plugins have modified settings
    plugin_settings = {}

    for plugin_instance in registered_plugins:
        _apply_base_settings(plugin_instance, settings_module)
        plugin_settings_module = plugin_instance.settings_module
        if plugin_settings_module:
            _process_module_settings(
                plugin_settings_module,
                plugin_settings,
                existing_settings,
                plugin_instance.module_path,
                settings_module,
            )
<|endoftext|>


<commit_before>class ControlFileParams:
    generalParams = "GeneralParams"
    spawningBlockname = "SpawningParams"
    simulationBlockname = "SimulationParams"
    clusteringBlockname = "clusteringTypes"

class GeneralParams:
    mandatory = {
        "restart": "bool",
        "outputPath": "unicode",
        "initialStructures": "list",
        "debug": "bool",
        "writeAllClusteringStructures": "bool"
    }
    params = {
        "restart": "bool",
        "outputPath": "unicode",
        "initialStructures": "list",
        "debug": "bool",
        "writeAllClusteringStructures": "bool",
        "nativeStructure": "unicode"
    }

class SpawningParams:
    params = {
        "epsilon": "numbers.Real",
        "T": "numbers.Real",
        "reportFilename": "unicode",
        "metricColumnInReport": "numbers.Real",
        "varEpsilonType": "unicode",
        "maxEpsilon": "numbers.Real",
        "minEpsilon": "numbers.Real",
        "variationWindow": "numbers.Real",
        "maxEpsilonWindow": "numbers.Real",
        "period": "numbers.Real",
        "metricWeights": "unicode"
    }
    types = {
        "sameWeight": {},
        "inverselyProportional": {},
        "epsilon": {
            "epsilon": "numbers.Real",
            "T": "numbers.Real",
            "reportFilename": "unicode",
            "metricColumnInReport": "numbers.Real",
        },
        "FAST": {
            "epsilon": "numbers.Real",
            "T": "numbers.Real",
            "reportFilename": "unicode",
            "metricColumnInReport": "numbers.Real",
        },
        "variableEpsilon": {
            "epsilon": "numbers.Real",
            "T": "numbers.Real",
            "reportFilename": "unicode",
            "metricColumnInReport": "numbers.Real",
            "varEpsilonType": "unicode",
            "maxEpsilon": "numbers.Real"
        }
    }
    density = {
        "types": {
            "heaviside": "unicode",
            "null": "unicode",
            "constant": "unicode",
            "continuous": "unicode"
        },
        "params": {
            "heaviside": "unicode",
            "null": "unicode",
            "constant": "unicode",
            "values": "list",
            "conditions": "list",
            "continuous": "unicode"
        }
    }

class SimulationParams:
    types = {"pele": {
                "processors": "numbers.Real",
                "controlFile": "unicode",
                "seed": "numbers.Real",
                "peleSteps": "numbers.Real",
                "iterations": "numbers.Real"
    },
             "test": {
                "destination": "unicode",
                "origin": "unicode",
                "processors": "numbers.Real",
                "seed": "numbers.Real",
                "peleSteps": "numbers.Real",
                "iterations": "numbers.Real"
             },
             "md": {}}
    params = {
        "executable": "unicode",
        "data": "unicode",
        "documents": "unicode",
        "destination": "unicode",
        "origin": "unicode",
        "processors": "numbers.Real",
        "controlFile": "unicode",
        "seed": "numbers.Real",
        "peleSteps": "numbers.Real",
        "iterations": "numbers.Real",
        "exitCondition": "dict"
    }
    exitCondition = {
        "types": {
            "metric": "unicode"
        },
        "params": {
            "metricCol": "int",
            "exitValue": "numbers.Real"
        }
    }

class clusteringTypes:
    types = {
        "contacts": {
            "ligandResname": "unicode",
            "contactThresholdDistance": "numbers.Real",
        },
        "contactMapAffinity": {
            "ligandResname": "unicode",
            "contactThresholdDistance": "numbers.Real",
        },
        "contactMapAgglomerative": {
            "ligandResname": "unicode",
            "contactThresholdDistance": "numbers.Real",
            "nclusters": "numbers.Real"
        },
        "contactMapAccumulative": {
            "ligandResname": "unicode",
            "contactThresholdDistance": "numbers.Real",
            "similarityEvaluator": "unicode"
        },
        "lastSnapshot": {
            "ligandResname": "unicode",
            "contactThresholdDistance": "numbers.Real",
        }
    }
    params = {
        "contacts": "unicode",
        "contactMapAffinity": "unicode",
        "contactMapAgglomerative": "unicode",
        "contactMapAccumaltive": "unicode",
        "lastSnapshot": "unicode",
        "ligandResname": "unicode",
        "contactThresholdDistance": "numbers.Real",
        "similarityEvaluator": "unicode",
        "symmetries": "list",
        "nclusters": "numbers.Real"
    }
    thresholdCalculator = {
        "types": {
            "heaviside": "unicode",
            "constant": "unicode"
        },
        "params": {
            "conditions": "list",
            "values": "list",
            "value": "numbers.Real",
            "heaviside": "unicode",
            "constant": "unicode"
        }
    }
<commit_msg>Validator now supports unicode and normal strings interchangebly<commit_after>class ControlFileParams:
    generalParams = "GeneralParams"
    spawningBlockname = "SpawningParams"
    simulationBlockname = "SimulationParams"
    clusteringBlockname = "clusteringTypes"

class GeneralParams:
    mandatory = {
        "restart": "bool",
        "outputPath": "basestring",
        "initialStructures": "list",
        "debug": "bool",
        "writeAllClusteringStructures": "bool"
    }
    params = {
        "restart": "bool",
        "outputPath": "basestring",
        "initialStructures": "list",
        "debug": "bool",
        "writeAllClusteringStructures": "bool",
        "nativeStructure": "basestring"
    }

class SpawningParams:
    params = {
        "epsilon": "numbers.Real",
        "T": "numbers.Real",
        "reportFilename": "basestring",
        "metricColumnInReport": "numbers.Real",
        "varEpsilonType": "basestring",
        "maxEpsilon": "numbers.Real",
        "minEpsilon": "numbers.Real",
        "variationWindow": "numbers.Real",
        "maxEpsilonWindow": "numbers.Real",
        "period": "numbers.Real",
        "metricWeights": "basestring"
    }
    types = {
        "sameWeight": {},
        "inverselyProportional": {},
        "epsilon": {
            "epsilon": "numbers.Real",
            "T": "numbers.Real",
            "reportFilename": "basestring",
            "metricColumnInReport": "numbers.Real",
        },
        "FAST": {
            "epsilon": "numbers.Real",
            "T": "numbers.Real",
            "reportFilename": "basestring",
            "metricColumnInReport": "numbers.Real",
        },
        "variableEpsilon": {
            "epsilon": "numbers.Real",
            "T": "numbers.Real",
            "reportFilename": "basestring",
            "metricColumnInReport": "numbers.Real",
            "varEpsilonType": "basestring",
            "maxEpsilon": "numbers.Real"
        }
    }
    density = {
        "types": {
            "heaviside": "basestring",
            "null": "basestring",
            "constant": "basestring",
            "continuous": "basestring"
        },
        "params": {
            "heaviside": "basestring",
            "null": "basestring",
            "constant": "basestring",
            "values": "list",
            "conditions": "list",
            "continuous": "basestring"
        }
    }

class SimulationParams:
    types = {"pele": {
                "processors": "numbers.Real",
                "controlFile": "basestring",
                "seed": "numbers.Real",
                "peleSteps": "numbers.Real",
                "iterations": "numbers.Real"
    },
             "test": {
                "destination": "basestring",
                "origin": "basestring",
                "processors": "numbers.Real",
                "seed": "numbers.Real",
                "peleSteps": "numbers.Real",
                "iterations": "numbers.Real"
             },
             "md": {}}
    params = {
        "executable": "basestring",
        "data": "basestring",
        "documents": "basestring",
        "destination": "basestring",
        "origin": "basestring",
        "processors": "numbers.Real",
        "controlFile": "basestring",
        "seed": "numbers.Real",
        "peleSteps": "numbers.Real",
        "iterations": "numbers.Real",
        "exitCondition": "dict"
    }
    exitCondition = {
        "types": {
            "metric": "basestring"
        },
        "params": {
            "metricCol": "int",
            "exitValue": "numbers.Real"
        }
    }

class clusteringTypes:
    types = {
        "contacts": {
            "ligandResname": "basestring",
            "contactThresholdDistance": "numbers.Real",
        },
        "contactMapAffinity": {
            "ligandResname": "basestring",
            "contactThresholdDistance": "numbers.Real",
        },
        "contactMapAgglomerative": {
            "ligandResname": "basestring",
            "contactThresholdDistance": "numbers.Real",
            "nclusters": "numbers.Real"
        },
        "contactMapAccumulative": {
            "ligandResname": "basestring",
            "contactThresholdDistance": "numbers.Real",
            "similarityEvaluator": "basestring"
        },
        "lastSnapshot": {
            "ligandResname": "basestring",
            "contactThresholdDistance": "numbers.Real",
        }
    }
    params = {
        "contacts": "basestring",
        "contactMapAffinity": "basestring",
        "contactMapAgglomerative": "basestring",
        "contactMapAccumaltive": "basestring",
        "lastSnapshot": "basestring",
        "ligandResname": "basestring",
        "contactThresholdDistance": "numbers.Real",
        "similarityEvaluator": "basestring",
        "symmetries": "list",
        "nclusters": "numbers.Real"
    }
    thresholdCalculator = {
        "types": {
            "heaviside": "basestring",
            "constant": "basestring"
        },
        "params": {
            "conditions": "list",
            "values": "list",
            "value": "numbers.Real",
            "heaviside": "basestring",
            "constant": "basestring"
        }
    }
<|endoftext|>