<commit_before>"""distutils.bcppcompiler

Contains BorlandCCompiler, an implementation of the abstract CCompiler class
for the Borland C++ compiler.
"""

# This implementation by Lyle Johnson, based on the original msvccompiler.py
# module and using the directions originally published by Gordon Williams.

# XXX looks like there's a LOT of overlap between these two classes:
# someone should sit down and factor out the common code as
# WindowsCCompiler!  --GPW

# XXX Lyle reports that this doesn't quite work yet:
# """...but this is what I've got so far. The compile step works fine but
# when it runs the link step I get an "out of memory" failure.  Since
# spawn() echoes the command it's trying to spawn, I can type the link line
# verbatim at the DOS prompt and it links the Windows DLL correctly -- so
# the syntax is correct. There's just some weird interaction going on when
# it tries to "spawn" the link process from within the setup.py script. I'm
# not really sure how to debug this one right off-hand; obviously there's
# nothing wrong with the "spawn()" function since it's working properly for
# the compile stage."""

__revision__ = "$Id$"

import sys, os, string
from distutils.errors import \
     DistutilsExecError, DistutilsPlatformError, \
     CompileError, LibError, LinkError
from distutils.ccompiler import \
     CCompiler, gen_preprocess_options, gen_lib_options

class BCPPCompiler(CCompiler) :
    """Concrete class that implements an interface to the Borland C/C++
    compiler, as defined by the CCompiler abstract class.
    """

    compiler_type = 'bcpp'

    # Just set this so CCompiler's constructor doesn't barf.  We currently
    # don't use the 'set_executables()' bureaucracy provided by CCompiler,
    # as it really isn't necessary for this sort of single-compiler class.
    # Would be nice to have a consistent interface with UnixCCompiler,
    # though, so it's worth thinking about.
    executables = {}

    # Private class data (need to distinguish C from C++ source for compiler)
    _c_extensions = ['.c']
    _cpp_extensions = ['.cc', '.cpp', '.cxx']

    # Needed for the filename generation methods provided by the
    # base class, CCompiler.
    src_extensions = _c_extensions + _cpp_extensions
    obj_extension = '.obj'
    static_lib_extension = '.lib'
    shared_lib_extension = '.dll'
    static_lib_format = shared_lib_format = '%s%s'
    exe_extension = '.exe'

    def __init__ (self,
                  verbose=0,
                  dry_run=0,
                  force=0):

        CCompiler.__init__ (self, verbose, dry_run, force)

        # These executables are assumed to all be in the path.
        # Borland doesn't seem to use any special registry settings to
        # indicate their installation locations.

        self.cc = "bcc32.exe"
        self.link = "ilink32.exe"
        self.lib = "tlib.exe"

        self.preprocess_options = None
        self.compile_options = ['/tWM', '/O2', '/q']
        self.compile_options_debug = ['/tWM', '/Od', '/q']

        self.ldflags_shared = ['/Tpd', '/Gn', '/q', '/x']
        self.ldflags_shared_debug = ['/Tpd', '/Gn', '/q', '/x']
        self.ldflags_static = []

    # -- Worker methods ------------------------------------------------

    def compile (self,
                 sources,
                 output_dir=None,
                 macros=None,
                 include_dirs=None,
                 debug=0,
                 extra_preargs=None,
                 extra_postargs=None):

        (output_dir, macros, include_dirs) = \
            self._fix_compile_args (output_dir, macros, include_dirs)
        (objects, skip_sources) = self._prep_compile (sources, output_dir)

        if extra_postargs is None:
            extra_postargs = []

        pp_opts = gen_preprocess_options (macros, include_dirs)
        compile_opts = extra_preargs or []
        compile_opts.append ('-c')
        if debug:
            compile_opts.extend (self.compile_options_debug)
        else:
            compile_opts.extend (self.compile_options)

        for i in range (len (sources)):
            src = sources[i] ; obj = objects[i]
            ext = (os.path.splitext (src))[1]

            if skip_sources[src]:
                self.announce ("skipping %s (%s up-to-date)" % (src, obj))
            else:
                if ext in self._c_extensions:
                    input_opt = ""
                elif ext in self._cpp_extensions:
                    input_opt = "-P"

                output_opt = "-o" + obj

                self.mkpath (os.path.dirname (obj))

                # Compiler command line syntax is: "bcc32 [options] file(s)".
                # Note that the source file names must appear at the end of
                # the command line.

                try:
                    self.spawn ([self.cc] + compile_opts + pp_opts +
                                [input_opt, output_opt] +
                                extra_postargs + [src])
                except DistutilsExecError, msg:
                    raise CompileError, msg

        return objects

    # compile ()

    def create_static_lib (self,
                           objects,
                           output_libname,
                           output_dir=None,
                           debug=0,
                           extra_preargs=None,
                           extra_postargs=None):

        (objects, output_dir) = self._fix_object_args (objects, output_dir)
        output_filename = \
            self.library_filename (output_libname, output_dir=output_dir)

        if self._need_link (objects, output_filename):
            lib_args = [output_filename, '/u'] + objects
            if debug:
                pass                    # XXX what goes here?
            if extra_preargs:
                lib_args[:0] = extra_preargs
            if extra_postargs:
                lib_args.extend (extra_postargs)
            try:
               self.spawn ([self.lib] + lib_args)
            except DistutilsExecError, msg:
               raise LibError, msg
        else:
            self.announce ("skipping %s (up-to-date)" % output_filename)

    # create_static_lib ()

    def link_shared_lib (self,
                         objects,
                         output_libname,
                         output_dir=None,
                         libraries=None,
                         library_dirs=None,
                         runtime_library_dirs=None,
                         export_symbols=None,
                         debug=0,
                         extra_preargs=None,
                         extra_postargs=None,
                         build_temp=None):

        self.link_shared_object (objects,
                                 self.shared_library_name(output_libname),
                                 output_dir=output_dir,
                                 libraries=libraries,
                                 library_dirs=library_dirs,
                                 runtime_library_dirs=runtime_library_dirs,
                                 export_symbols=export_symbols,
                                 debug=debug,
                                 extra_preargs=extra_preargs,
                                 extra_postargs=extra_postargs,
                                 build_temp=build_temp)

    def link_shared_object (self,
                            objects,
                            output_filename,
                            output_dir=None,
                            libraries=None,
                            library_dirs=None,
                            runtime_library_dirs=None,
                            export_symbols=None,
                            debug=0,
                            extra_preargs=None,
                            extra_postargs=None,
                            build_temp=None):

        (objects, output_dir) = self._fix_object_args (objects, output_dir)
        (libraries, library_dirs, runtime_library_dirs) = \
            self._fix_lib_args (libraries, library_dirs, runtime_library_dirs)

        if runtime_library_dirs:
            self.warn ("I don't know what to do with 'runtime_library_dirs': "
                       + str (runtime_library_dirs))

        if output_dir is not None:
            output_filename = os.path.join (output_dir, output_filename)

        if self._need_link (objects, output_filename):

            if debug:
                ldflags = self.ldflags_shared_debug
            else:
                ldflags = self.ldflags_shared

            startup_obj = 'c0d32'

            libraries.append ('mypylib')
            libraries.append ('import32')
            libraries.append ('cw32mt')

            # Create a temporary exports file for use by the linker
            head, tail = os.path.split (output_filename)
            modname, ext = os.path.splitext (tail)
            def_file = os.path.join (build_temp, '%s.def' % modname)
            f = open (def_file, 'w')
            f.write ('EXPORTS\n')
            for sym in (export_symbols or []):
                f.write ('  %s=_%s\n' % (sym, sym))

            ld_args = ldflags + [startup_obj] + objects + \
                [',%s,,' % output_filename] + \
                libraries + [',' + def_file]

            if extra_preargs:
                ld_args[:0] = extra_preargs
            if extra_postargs:
                ld_args.extend (extra_postargs)

            self.mkpath (os.path.dirname (output_filename))
            try:
                self.spawn ([self.link] + ld_args)
            except DistutilsExecError, msg:
                raise LinkError, msg

        else:
            self.announce ("skipping %s (up-to-date)" % output_filename)

    # link_shared_object ()

    def link_executable (self,
                         objects,
                         output_progname,
                         output_dir=None,
                         libraries=None,
                         library_dirs=None,
                         runtime_library_dirs=None,
                         debug=0,
                         extra_preargs=None,
                         extra_postargs=None):

        (objects, output_dir) = self._fix_object_args (objects, output_dir)
        (libraries, library_dirs, runtime_library_dirs) = \
            self._fix_lib_args (libraries, library_dirs, runtime_library_dirs)

        if runtime_library_dirs:
            self.warn ("I don't know what to do with 'runtime_library_dirs': "
                       + str (runtime_library_dirs))

        lib_opts = gen_lib_options (self,
                                    library_dirs, runtime_library_dirs,
                                    libraries)
        output_filename = output_progname + self.exe_extension
        if output_dir is not None:
            output_filename = os.path.join (output_dir, output_filename)

        if self._need_link (objects, output_filename):

            if debug:
                ldflags = self.ldflags_shared_debug[1:]
            else:
                ldflags = self.ldflags_shared[1:]

            ld_args = ldflags + lib_opts + \
                      objects + ['/OUT:' + output_filename]

            if extra_preargs:
                ld_args[:0] = extra_preargs
            if extra_postargs:
                ld_args.extend (extra_postargs)

            self.mkpath (os.path.dirname (output_filename))
            try:
                self.spawn ([self.link] + ld_args)
            except DistutilsExecError, msg:
                raise LinkError, msg
        else:
            self.announce ("skipping %s (up-to-date)" % output_filename)

    # -- Miscellaneous methods -----------------------------------------
    # These are all used by the 'gen_lib_options() function, in
    # ccompiler.py.

    def library_dir_option (self, dir):
        return "-L" + dir

    def runtime_library_dir_option (self, dir):
        raise DistutilsPlatformError, \
              "don't know how to set runtime library search path for MSVC++"

    def library_option (self, lib):
        return self.library_filename (lib)

    def find_library_file (self, dirs, lib):

        for dir in dirs:
            libfile = os.path.join (dir, self.library_filename (lib))
            if os.path.exists (libfile):
                return libfile

        else:
            # Oops, didn't find it in *any* of 'dirs'
            return None

<commit_msg>Patch from Rene Liebscher.  Some ugly changes, but supposedly this makes it so BCPPCompiler actually works, so I'm provisionally accepting it -- ugly and working is better than not working!  Major changes:   - normalize paths (apparently BC++ doesn't like slashes)   - overhauled how we search for and specify libraries on the linker     command-line   - hacked up 'find_library_file()' so it knows about "debug" library     naming convention as well as "bcpp_xxx.lib" -- the question is,     is this a well-established and sensible convention? Also:   - change to use 'util.write_file()' to write the .def file<commit_after>"""distutils.bcppcompiler

Contains BorlandCCompiler, an implementation of the abstract CCompiler class
for the Borland C++ compiler.
"""

# This implementation by Lyle Johnson, based on the original msvccompiler.py
# module and using the directions originally published by Gordon Williams.

# XXX looks like there's a LOT of overlap between these two classes:
# someone should sit down and factor out the common code as
# WindowsCCompiler!  --GPW

__revision__ = "$Id$"

import sys, os, string
from distutils.errors import \
     DistutilsExecError, DistutilsPlatformError, \
     CompileError, LibError, LinkError
from distutils.ccompiler import \
     CCompiler, gen_preprocess_options, gen_lib_options
from distutils.file_util import write_file

class BCPPCompiler(CCompiler) :
    """Concrete class that implements an interface to the Borland C/C++
    compiler, as defined by the CCompiler abstract class.
    """

    compiler_type = 'bcpp'

    # Just set this so CCompiler's constructor doesn't barf.  We currently
    # don't use the 'set_executables()' bureaucracy provided by CCompiler,
    # as it really isn't necessary for this sort of single-compiler class.
    # Would be nice to have a consistent interface with UnixCCompiler,
    # though, so it's worth thinking about.
    executables = {}

    # Private class data (need to distinguish C from C++ source for compiler)
    _c_extensions = ['.c']
    _cpp_extensions = ['.cc', '.cpp', '.cxx']

    # Needed for the filename generation methods provided by the
    # base class, CCompiler.
    src_extensions = _c_extensions + _cpp_extensions
    obj_extension = '.obj'
    static_lib_extension = '.lib'
    shared_lib_extension = '.dll'
    static_lib_format = shared_lib_format = '%s%s'
    exe_extension = '.exe'

    def __init__ (self,
                  verbose=0,
                  dry_run=0,
                  force=0):

        CCompiler.__init__ (self, verbose, dry_run, force)

        # These executables are assumed to all be in the path.
        # Borland doesn't seem to use any special registry settings to
        # indicate their installation locations.

        self.cc = "bcc32.exe"
        self.link = "ilink32.exe"
        self.lib = "tlib.exe"

        self.preprocess_options = None
        self.compile_options = ['/tWM', '/O2', '/q']
        self.compile_options_debug = ['/tWM', '/Od', '/q']

        self.ldflags_shared = ['/Tpd', '/Gn', '/q', '/x']
        self.ldflags_shared_debug = ['/Tpd', '/Gn', '/q', '/x']
        self.ldflags_static = []

    # -- Worker methods ------------------------------------------------

    def compile (self,
                 sources,
                 output_dir=None,
                 macros=None,
                 include_dirs=None,
                 debug=0,
                 extra_preargs=None,
                 extra_postargs=None):

        (output_dir, macros, include_dirs) = \
            self._fix_compile_args (output_dir, macros, include_dirs)
        (objects, skip_sources) = self._prep_compile (sources, output_dir)

        if extra_postargs is None:
            extra_postargs = []

        pp_opts = gen_preprocess_options (macros, include_dirs)
        compile_opts = extra_preargs or []
        compile_opts.append ('-c')
        if debug:
            compile_opts.extend (self.compile_options_debug)
        else:
            compile_opts.extend (self.compile_options)

        for i in range (len (sources)):
            src = sources[i] ; obj = objects[i]
            ext = (os.path.splitext (src))[1]

            if skip_sources[src]:
                self.announce ("skipping %s (%s up-to-date)" % (src, obj))
            else:
                if ext in self._c_extensions:
                    input_opt = ""
                elif ext in self._cpp_extensions:
                    input_opt = "-P"

                src = os.path.normpath(src)
                obj = os.path.normpath(obj)

                output_opt = "-o" + obj
                self.mkpath(os.path.dirname(obj))

                # Compiler command line syntax is: "bcc32 [options] file(s)".
                # Note that the source file names must appear at the end of
                # the command line.
                try:
                    self.spawn ([self.cc] + compile_opts + pp_opts +
                                [input_opt, output_opt] +
                                extra_postargs + [src])
                except DistutilsExecError, msg:
                    raise CompileError, msg

        return objects

    # compile ()

    def create_static_lib (self,
                           objects,
                           output_libname,
                           output_dir=None,
                           debug=0,
                           extra_preargs=None,
                           extra_postargs=None):

        (objects, output_dir) = self._fix_object_args (objects, output_dir)
        output_filename = \
            self.library_filename (output_libname, output_dir=output_dir)

        if self._need_link (objects, output_filename):
            lib_args = [output_filename, '/u'] + objects
            if debug:
                pass                    # XXX what goes here?
            if extra_preargs:
                lib_args[:0] = extra_preargs
            if extra_postargs:
                lib_args.extend (extra_postargs)
            try:
               self.spawn ([self.lib] + lib_args)
            except DistutilsExecError, msg:
               raise LibError, msg
        else:
            self.announce ("skipping %s (up-to-date)" % output_filename)

    # create_static_lib ()

    def link_shared_lib (self,
                         objects,
                         output_libname,
                         output_dir=None,
                         libraries=None,
                         library_dirs=None,
                         runtime_library_dirs=None,
                         export_symbols=None,
                         debug=0,
                         extra_preargs=None,
                         extra_postargs=None,
                         build_temp=None):

        self.link_shared_object (objects,
                                 self.shared_library_name(output_libname),
                                 output_dir=output_dir,
                                 libraries=libraries,
                                 library_dirs=library_dirs,
                                 runtime_library_dirs=runtime_library_dirs,
                                 export_symbols=export_symbols,
                                 debug=debug,
                                 extra_preargs=extra_preargs,
                                 extra_postargs=extra_postargs,
                                 build_temp=build_temp)

    def link_shared_object (self,
                            objects,
                            output_filename,
                            output_dir=None,
                            libraries=None,
                            library_dirs=None,
                            runtime_library_dirs=None,
                            export_symbols=None,
                            debug=0,
                            extra_preargs=None,
                            extra_postargs=None,
                            build_temp=None):

        # XXX this ignores 'build_temp'!  should follow the lead of
        # msvccompiler.py

        (objects, output_dir) = self._fix_object_args (objects, output_dir)
        (libraries, library_dirs, runtime_library_dirs) = \
            self._fix_lib_args (libraries, library_dirs, runtime_library_dirs)

        if runtime_library_dirs:
            self.warn ("I don't know what to do with 'runtime_library_dirs': "
                       + str (runtime_library_dirs))

        if output_dir is not None:
            output_filename = os.path.join (output_dir, output_filename)

        if self._need_link (objects, output_filename):

            if debug:
                ld_args = self.ldflags_shared_debug[:]
            else:
                ld_args = self.ldflags_shared[:]

            # Borland C++ has problems with '/' in paths
            objects = map(os.path.normpath, objects)
            startup_obj = 'c0d32'
            objects.insert(0, startup_obj)

            # either exchange python15.lib in the python libs directory against
            # a Borland-like one, or create one with name bcpp_python15.lib
            # there and remove the pragmas from config.h
            #libraries.append ('mypylib')
            libraries.append ('import32')
            libraries.append ('cw32mt')

            # Create a temporary exports file for use by the linker
            head, tail = os.path.split (output_filename)
            modname, ext = os.path.splitext (tail)
            temp_dir = os.path.dirname(objects[0]) # preserve tree structure
            def_file = os.path.join (temp_dir, '%s.def' % modname)
            contents = ['EXPORTS']
            for sym in (export_symbols or []):
                contents.append('  %s=_%s' % (sym, sym))
            self.execute(write_file, (def_file, contents),
                         "writing %s" % def_file)

            # Start building command line flags and options.

            for l in library_dirs:
                ld_args.append("/L%s" % os.path.normpath(l))

            ld_args.extend(objects)     # list of object files

            # name of dll file
            ld_args.extend([',',output_filename])
            # no map file and start libraries
            ld_args.extend([',', ','])

            for lib in libraries:
                # see if we find it and if there is a bcpp specific lib
                # (bcpp_xxx.lib)
                libfile = self.find_library_file(library_dirs, lib, debug)
                if libfile is None:
                    ld_args.append(lib)
                    # probably a BCPP internal library -- don't warn
                    #    self.warn('library %s not found.' % lib)
                else:
                    # full name which prefers bcpp_xxx.lib over xxx.lib
                    ld_args.append(libfile)
            # def file for export symbols
            ld_args.extend([',',def_file])

            if extra_preargs:
                ld_args[:0] = extra_preargs
            if extra_postargs:
                ld_args.extend(extra_postargs)

            self.mkpath (os.path.dirname (output_filename))
            try:
                self.spawn ([self.link] + ld_args)
            except DistutilsExecError, msg:
                raise LinkError, msg

        else:
            self.announce ("skipping %s (up-to-date)" % output_filename)

    # link_shared_object ()

    def link_executable (self,
                         objects,
                         output_progname,
                         output_dir=None,
                         libraries=None,
                         library_dirs=None,
                         runtime_library_dirs=None,
                         debug=0,
                         extra_preargs=None,
                         extra_postargs=None):

        (objects, output_dir) = self._fix_object_args (objects, output_dir)
        (libraries, library_dirs, runtime_library_dirs) = \
            self._fix_lib_args (libraries, library_dirs, runtime_library_dirs)

        if runtime_library_dirs:
            self.warn ("I don't know what to do with 'runtime_library_dirs': "
                       + str (runtime_library_dirs))

        lib_opts = gen_lib_options (self,
                                    library_dirs, runtime_library_dirs,
                                    libraries)
        output_filename = output_progname + self.exe_extension
        if output_dir is not None:
            output_filename = os.path.join (output_dir, output_filename)

        if self._need_link (objects, output_filename):

            if debug:
                ldflags = self.ldflags_shared_debug[1:]
            else:
                ldflags = self.ldflags_shared[1:]

            ld_args = ldflags + lib_opts + \
                      objects + ['/OUT:' + output_filename]

            if extra_preargs:
                ld_args[:0] = extra_preargs
            if extra_postargs:
                ld_args.extend (extra_postargs)

            self.mkpath (os.path.dirname (output_filename))
            try:
                self.spawn ([self.link] + ld_args)
            except DistutilsExecError, msg:
                raise LinkError, msg
        else:
            self.announce ("skipping %s (up-to-date)" % output_filename)

    # -- Miscellaneous methods -----------------------------------------
    # These are all used by the 'gen_lib_options() function, in
    # ccompiler.py.

    def library_dir_option (self, dir):
        return "-L" + dir

    def runtime_library_dir_option (self, dir):
        raise DistutilsPlatformError, \
              ("don't know how to set runtime library search path "
               "for Borland C++")

    def library_option (self, lib):
        return self.library_filename (lib)

    def find_library_file (self, dirs, lib, debug=0):
        # find library file
        # bcpp_xxx.lib is better than xxx.lib
        # and xxx_d.lib is better than xxx.lib if debug is set
        for dir in dirs:
            if debug:
                libfile = os.path.join (
                    dir, self.library_filename ("bcpp_" + lib + "_d"))
                if os.path.exists (libfile):
                    return libfile
            libfile = os.path.join (
                dir, self.library_filename ("bcpp_" + lib))
            if os.path.exists (libfile):
                return libfile
            if debug:
                libfile = os.path.join (
                    dir, self.library_filename(lib + '_d'))
                if os.path.exists (libfile):
                    return libfile
            libfile = os.path.join (dir, self.library_filename (lib))
            if os.path.exists (libfile):
                return libfile

        else:
            # Oops, didn't find it in *any* of 'dirs'
            return None

<|endoftext|>


<commit_before>import Queue
import StringIO
import base64
import json
import os.path
import shutil
import tarfile
import threading

from sh import git  # pylint: disable=no-name-in-module

class SubmissionGitSaver(threading.Thread):

    """
        Thread class that saves results from submission in the git repo.
        It must be a thread as a git commit can take some time and because we extract archives returned by Job Manager.
        But it must also be launched only one time as our git operations are not really process/tread-safe ;-)
    """

    def __init__(self, plugin_manager, config):
        threading.Thread.__init__(self)
        self.queue = Queue.Queue()
        mustdoinit = False
        self.repopath = config.get("repo_directory", "./repo_submissions")
        if not os.path.exists(self.repopath):
            mustdoinit = True
            os.mkdir(self.repopath)
        self.git = git.bake('--work-tree=' + self.repopath, '--git-dir=' + os.path.join(self.repopath, '.git'))
        if mustdoinit:
            self.git.init()
        plugin_manager.add_hook('submission_done', self.add)
        print "SubmissionGitSaver started"

    def add(self, submission, job):
        """ Add a new submission to the repo (add the to queue, will be saved async)"""
        self.queue.put((submission, job))

    def run(self):
        while True:
            try:
                submission, job = self.queue.get()
                self.save(submission, job)
            except Exception as inst:
                print "Exception in JobSaver: " + str(inst)
                pass

    def save(self, submission, job):
        """ saves a new submission in the repo (done async) """
        # Save submission to repo
        print "Save submission " + str(submission["_id"]) + " to git repo"
        # Verify that the directory for the course exists
        if not os.path.exists(os.path.join(self.repopath, submission["courseid"])):
            os.mkdir(os.path.join(self.repopath, submission["courseid"]))
        # Idem with the task
        if not os.path.exists(os.path.join(self.repopath, submission["courseid"], submission["taskid"])):
            os.mkdir(os.path.join(self.repopath, submission["courseid"], submission["taskid"]))
        # Idem with the username, but empty it
        dirname = os.path.join(self.repopath, submission["courseid"], submission["taskid"], submission["username"])
        if os.path.exists(dirname):
            shutil.rmtree(dirname)
        os.mkdir(dirname)
        # Now we can put the input, the output and the zip
        open(os.path.join(dirname, 'submitted_on'), "w+").write(str(submission["submitted_on"]))
        open(os.path.join(dirname, 'input.json'), "w+").write(json.dumps(submission["input"]))
        result_obj = {
            "result": job["result"],
            "text": (job["text"] if "text" in job else None),
            "problems": (job["problems"] if "problems" in job else {})
        }
        open(os.path.join(dirname, 'result.json'), "w+").write(json.dumps(result_obj))
        if "archive" in job:
            os.mkdir(os.path.join(dirname, 'output'))
            tar = tarfile.open(mode='r:gz', fileobj=StringIO.StringIO(base64.b64decode(job["archive"])))
            tar.extractall(os.path.join(dirname, 'output'))
            tar.close()

        self.git.add('--all', '.')
        title = " - ".join([str(submission["courseid"]) + "/" + str(submission["taskid"]),
                            str(submission["_id"]),
                            submission["username"],
                            ("success" if "result" in job and job["result"] == "success" else "failed")])
        self.git.commit('-m', title)

def init(plugin_manager, config):
    """
        Init the plugin

        Available configuration:
        ::

            {
                "plugin_module": "frontend.plugins.git_repo",
                "repo_directory": "./repo_submissions"
            }

    """
    submission_git_saver = SubmissionGitSaver(plugin_manager, config)
    submission_git_saver.daemon = True
    submission_git_saver.start()
<commit_msg>Fix pass...<commit_after>import Queue
import StringIO
import base64
import json
import os.path
import shutil
import tarfile
import threading

from sh import git  # pylint: disable=no-name-in-module

class SubmissionGitSaver(threading.Thread):

    """
        Thread class that saves results from submission in the git repo.
        It must be a thread as a git commit can take some time and because we extract archives returned by Job Manager.
        But it must also be launched only one time as our git operations are not really process/tread-safe ;-)
    """

    def __init__(self, plugin_manager, config):
        threading.Thread.__init__(self)
        self.queue = Queue.Queue()
        mustdoinit = False
        self.repopath = config.get("repo_directory", "./repo_submissions")
        if not os.path.exists(self.repopath):
            mustdoinit = True
            os.mkdir(self.repopath)
        self.git = git.bake('--work-tree=' + self.repopath, '--git-dir=' + os.path.join(self.repopath, '.git'))
        if mustdoinit:
            self.git.init()
        plugin_manager.add_hook('submission_done', self.add)
        print "SubmissionGitSaver started"

    def add(self, submission, job):
        """ Add a new submission to the repo (add the to queue, will be saved async)"""
        self.queue.put((submission, job))

    def run(self):
        while True:
            try:
                submission, job = self.queue.get()
                self.save(submission, job)
            except Exception as inst:
                print "Exception in JobSaver: " + str(inst)

    def save(self, submission, job):
        """ saves a new submission in the repo (done async) """
        # Save submission to repo
        print "Save submission " + str(submission["_id"]) + " to git repo"
        # Verify that the directory for the course exists
        if not os.path.exists(os.path.join(self.repopath, submission["courseid"])):
            os.mkdir(os.path.join(self.repopath, submission["courseid"]))
        # Idem with the task
        if not os.path.exists(os.path.join(self.repopath, submission["courseid"], submission["taskid"])):
            os.mkdir(os.path.join(self.repopath, submission["courseid"], submission["taskid"]))
        # Idem with the username, but empty it
        dirname = os.path.join(self.repopath, submission["courseid"], submission["taskid"], submission["username"])
        if os.path.exists(dirname):
            shutil.rmtree(dirname)
        os.mkdir(dirname)
        # Now we can put the input, the output and the zip
        open(os.path.join(dirname, 'submitted_on'), "w+").write(str(submission["submitted_on"]))
        open(os.path.join(dirname, 'input.json'), "w+").write(json.dumps(submission["input"]))
        result_obj = {
            "result": job["result"],
            "text": (job["text"] if "text" in job else None),
            "problems": (job["problems"] if "problems" in job else {})
        }
        open(os.path.join(dirname, 'result.json'), "w+").write(json.dumps(result_obj))
        if "archive" in job:
            os.mkdir(os.path.join(dirname, 'output'))
            tar = tarfile.open(mode='r:gz', fileobj=StringIO.StringIO(base64.b64decode(job["archive"])))
            tar.extractall(os.path.join(dirname, 'output'))
            tar.close()

        self.git.add('--all', '.')
        title = " - ".join([str(submission["courseid"]) + "/" + str(submission["taskid"]),
                            str(submission["_id"]),
                            submission["username"],
                            ("success" if "result" in job and job["result"] == "success" else "failed")])
        self.git.commit('-m', title)

def init(plugin_manager, config):
    """
        Init the plugin

        Available configuration:
        ::

            {
                "plugin_module": "frontend.plugins.git_repo",
                "repo_directory": "./repo_submissions"
            }

    """
    submission_git_saver = SubmissionGitSaver(plugin_manager, config)
    submission_git_saver.daemon = True
    submission_git_saver.start()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

#
# OSV: Objects Services
#

import orm
import netsvc
import pooler
import copy
import sys
import traceback
import logging
from psycopg2 import IntegrityError, errorcodes
from tools.func import wraps

module_list = []
module_class_list = {}
class_pool = {}

class except_osv(Exception):
    def __init__(self, name, value, exc_type='warning'):
        self.name = name
        self.exc_type = exc_type
        self.value = value
        self.args = (exc_type, name)

class osv_pool(netsvc.Service):

    def check(f):
        @wraps(f)
        def wrapper(self, dbname, *args, **kwargs):
            try:
                if not pooler.get_pool(dbname)._ready:
                    raise except_osv('Database not ready', 'Currently, this database is not fully loaded and can not be used.')
                return f(self, dbname, *args, **kwargs)
            except orm.except_orm, inst:
                if inst.name == 'AccessError':
                    self.logger.debug("AccessError", exc_info=True)
                self.abortResponse(1, inst.name, 'warning', inst.value)
            except except_osv, inst:
                self.abortResponse(1, inst.name, inst.exc_type, inst.value)
            except IntegrityError, inst:
                for key in self._sql_error.keys():
                    if key in inst[0]:
                        self.abortResponse(1, 'Constraint Error', 'warning', self._sql_error[key])
                if inst.pgcode == errorcodes.NOT_NULL_VIOLATION:
                    msg = 'Sorry, this record cannot be deleted at the moment because other records still reference it.'
                    self.logger.debug("IntegrityError", exc_info=True)
                    try:
                        context = inst.pgerror.split('"public".')[1]
                        model_name = table = context.split('"')[1]
                        model = table.replace("_",".")
                        model_obj = self.get(model)
                        if model_obj:
                            model_name = model_obj._description or model_obj._name
                        msg += '\n\n[object with reference: %s - %s]' % (model_name, model)
                    except Exception:
                        pass
                    self.abortResponse(1, 'Integrity Error', 'warning', msg)
                else:
                    self.abortResponse(1, 'Integrity Error', 'warning', inst[0])
            except Exception, e:
                self.logger.exception("Uncaught exception")
                raise

        return wrapper

    def __init__(self):
        self._ready = False
        self.obj_pool = {}
        self.module_object_list = {}
        self.created = []
        self._sql_error = {}
        self._store_function = {}
        self._init = True
        self._init_parent = {}
        self.logger = logging.getLogger("web-services")
        netsvc.Service.__init__(self, 'object_proxy', audience='')
        self.exportMethod(self.obj_list)
        self.exportMethod(self.exec_workflow)
        self.exportMethod(self.execute)
        self.exportMethod(self.set_debug)

    def init_set(self, cr, mode):
        different = mode != self._init
        if different:
            if mode:
                self._init_parent = {}
            if not mode:
                for o in self._init_parent:
                    self.get(o)._parent_store_compute(cr)
            self._init = mode

        self._ready = True
        return different

    def execute_cr(self, cr, uid, obj, method, *args, **kw):
        object = pooler.get_pool(cr.dbname).get(obj)
        if not object:
            raise except_osv('Object Error', 'Object %s doesn\'t exist' % str(obj))
        return getattr(object, method)(cr, uid, *args, **kw)

    @check
    def execute(self, db, uid, obj, method, *args, **kw):
        db, pool = pooler.get_db_and_pool(db)
        cr = db.cursor()
        try:
            try:
                if method.startswith('_'):
                    raise except_osv('Access Denied', 'Private methods (such as %s) cannot be called remotely.' % (method,))
                res = pool.execute_cr(cr, uid, obj, method, *args, **kw)
                if res is None:
                    self.logger.warning('Method %s.%s can not return a None value (crash in XML-RPC)' % (obj, method))
                cr.commit()
            except Exception:
                cr.rollback()
                raise
        finally:
            cr.close()
        return res

    def exec_workflow_cr(self, cr, uid, obj, method, *args):
        wf_service = netsvc.LocalService("workflow")
        return wf_service.trg_validate(uid, obj, args[0], method, cr)

    @check
    def exec_workflow(self, db, uid, obj, method, *args):
        cr = pooler.get_db(db).cursor()
        try:
            try:
                res = self.exec_workflow_cr(cr, uid, obj, method, *args)
                cr.commit()
            except Exception:
                cr.rollback()
                raise
        finally:
            cr.close()
        return res

    def obj_list(self):
        return self.obj_pool.keys()

    # adds a new object instance to the object pool.
    # if it already existed, the instance is replaced
    def add(self, name, obj_inst):
        if name in self.obj_pool:
            del self.obj_pool[name]
        self.obj_pool[name] = obj_inst

        module = str(obj_inst.__class__)[6:]
        module = module[:len(module)-1]
        module = module.split('.')[0][2:]
        self.module_object_list.setdefault(module, []).append(obj_inst)

    # Return None if object does not exist
    def get(self, name):
        obj = self.obj_pool.get(name, None)
        return obj

    #TODO: pass a list of modules to load
    def instanciate(self, module, cr):
        res = []
        class_list = module_class_list.get(module, [])
        for klass in class_list:
            res.append(klass.createInstance(self, module, cr))
        return res

    def set_debug(self, db, obj, do_debug=True):
        object = pooler.get_pool(db).get(obj)
        if not object:
            raise except_osv('Object Error', 'Object %s doesn\'t exist' % str(obj))
        try:
            object._debug = do_debug
            return True
        except:
            raise

class osv_base(object):
    def __init__(self, pool, cr):
        pool.add(self._name, self)
        self.pool = pool
        super(osv_base, self).__init__(cr)

    def __new__(cls):
        module = str(cls)[6:]
        module = module[:len(module)-1]
        module = module.split('.')[0][2:]
        if not hasattr(cls, '_module'):
            cls._module = module
        module_class_list.setdefault(cls._module, []).append(cls)
        class_pool[cls._name] = cls
        if module not in module_list:
            module_list.append(cls._module)
        return None

class osv_memory(osv_base, orm.orm_memory):
    #
    # Goal: try to apply inheritancy at the instanciation level and
    #       put objects in the pool var
    #
    def createInstance(cls, pool, module, cr):
        parent_names = getattr(cls, '_inherit', None)
        if parent_names:
            if isinstance(parent_names, (str, unicode)):
                name = cls._name or parent_names
                parent_names = [parent_names]
            else:
                name = cls._name
            if not name:
                raise TypeError('_name is mandatory in case of multiple inheritance')

            for parent_name in ((type(parent_names)==list) and parent_names or [parent_names]):
                parent_class = pool.get(parent_name).__class__
                assert pool.get(parent_name), "parent class %s does not exist in module %s !" % (parent_name, module)
                nattr = {}
                for s in ('_columns', '_defaults'):
                    new = copy.copy(getattr(pool.get(parent_name), s))
                    if hasattr(new, 'update'):
                        new.update(cls.__dict__.get(s, {}))
                    else:
                        new.extend(cls.__dict__.get(s, []))
                    nattr[s] = new
                cls = type(name, (cls, parent_class), nattr)

        obj = object.__new__(cls)
        obj.__init__(pool, cr)
        return obj
    createInstance = classmethod(createInstance)

class osv(osv_base, orm.orm):
    #
    # Goal: try to apply inheritancy at the instanciation level and
    #       put objects in the pool var
    #
    def createInstance(cls, pool, module, cr):
        parent_names = getattr(cls, '_inherit', None)
        if parent_names:
            if isinstance(parent_names, (str, unicode)):
                name = cls._name or parent_names
                parent_names = [parent_names]
            else:
                name = cls._name
            if not name:
                raise TypeError('_name is mandatory in case of multiple inheritance')

            for parent_name in ((type(parent_names)==list) and parent_names or [parent_names]):
                parent_class = pool.get(parent_name).__class__
                assert pool.get(parent_name), "parent class %s does not exist in module %s !" % (parent_name, module)
                nattr = {}
                for s in ('_columns', '_defaults', '_inherits', '_constraints', '_sql_constraints'):
                    new = copy.copy(getattr(pool.get(parent_name), s))
                    if hasattr(new, 'update'):
                        new.update(cls.__dict__.get(s, {}))
                    else:
                        if s=='_constraints':
                            for c in cls.__dict__.get(s, []):
                                exist = False
                                for c2 in range(len(new)):
                                     #For _constraints, we should check field and methods as well
                                     if new[c2][2]==c[2] and new[c2][0]==c[0]:
                                        new[c2] = c
                                        exist = True
                                        break
                                if not exist:
                                    new.append(c)
                        else:
                            new.extend(cls.__dict__.get(s, []))
                    nattr[s] = new
                cls = type(name, (cls, parent_class), nattr)
        obj = object.__new__(cls)
        obj.__init__(pool, cr)
        return obj
    createInstance = classmethod(createInstance)

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<commit_msg>IMP: When calling a fn fails, try to inspect the code<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

#
# OSV: Objects Services
#

import orm
import netsvc
import pooler
import copy
import sys
import traceback
import logging
from psycopg2 import IntegrityError, errorcodes
from tools.func import wraps

module_list = []
module_class_list = {}
class_pool = {}

class except_osv(Exception):
    def __init__(self, name, value, exc_type='warning'):
        self.name = name
        self.exc_type = exc_type
        self.value = value
        self.args = (exc_type, name)

class osv_pool(netsvc.Service):

    def check(f):
        @wraps(f)
        def wrapper(self, dbname, *args, **kwargs):
            try:
                if not pooler.get_pool(dbname)._ready:
                    raise except_osv('Database not ready', 'Currently, this database is not fully loaded and can not be used.')
                return f(self, dbname, *args, **kwargs)
            except orm.except_orm, inst:
                if inst.name == 'AccessError':
                    self.logger.debug("AccessError", exc_info=True)
                self.abortResponse(1, inst.name, 'warning', inst.value)
            except except_osv, inst:
                self.abortResponse(1, inst.name, inst.exc_type, inst.value)
            except IntegrityError, inst:
                for key in self._sql_error.keys():
                    if key in inst[0]:
                        self.abortResponse(1, 'Constraint Error', 'warning', self._sql_error[key])
                if inst.pgcode == errorcodes.NOT_NULL_VIOLATION:
                    msg = 'Sorry, this record cannot be deleted at the moment because other records still reference it.'
                    self.logger.debug("IntegrityError", exc_info=True)
                    try:
                        context = inst.pgerror.split('"public".')[1]
                        model_name = table = context.split('"')[1]
                        model = table.replace("_",".")
                        model_obj = self.get(model)
                        if model_obj:
                            model_name = model_obj._description or model_obj._name
                        msg += '\n\n[object with reference: %s - %s]' % (model_name, model)
                    except Exception:
                        pass
                    self.abortResponse(1, 'Integrity Error', 'warning', msg)
                else:
                    self.abortResponse(1, 'Integrity Error', 'warning', inst[0])
            except Exception, e:
                self.logger.exception("Uncaught exception")
                raise

        return wrapper

    def __init__(self):
        self._ready = False
        self.obj_pool = {}
        self.module_object_list = {}
        self.created = []
        self._sql_error = {}
        self._store_function = {}
        self._init = True
        self._init_parent = {}
        self.logger = logging.getLogger("web-services")
        netsvc.Service.__init__(self, 'object_proxy', audience='')
        self.exportMethod(self.obj_list)
        self.exportMethod(self.exec_workflow)
        self.exportMethod(self.execute)
        self.exportMethod(self.set_debug)

    def init_set(self, cr, mode):
        different = mode != self._init
        if different:
            if mode:
                self._init_parent = {}
            if not mode:
                for o in self._init_parent:
                    self.get(o)._parent_store_compute(cr)
            self._init = mode

        self._ready = True
        return different

    def execute_cr(self, cr, uid, obj, method, *args, **kw):
        object = pooler.get_pool(cr.dbname).get(obj)
        if not object:
            raise except_osv('Object Error', 'Object %s doesn\'t exist' % str(obj))
        try:
            return getattr(object, method)(cr, uid, *args, **kw)
        except TypeError, e:
            try:
                import inspect
                fn = getattr(object, method)
                __fn_name = method
                __fn_file = inspect.getfile(fn)
                tb_s = "Function %s from file %s" %( __fn_name, __fn_file)
            except:
                tb_s = "Object: %s Function: %s\n" % (object, getattr(object, method))
            logger = Logger()
            logger.notifyChannel('web-services', LOG_ERROR, tb_s)
            raise

    @check
    def execute(self, db, uid, obj, method, *args, **kw):
        db, pool = pooler.get_db_and_pool(db)
        cr = db.cursor()
        try:
            try:
                if method.startswith('_'):
                    raise except_osv('Access Denied', 'Private methods (such as %s) cannot be called remotely.' % (method,))
                res = pool.execute_cr(cr, uid, obj, method, *args, **kw)
                if res is None:
                    self.logger.warning('Method %s.%s can not return a None value (crash in XML-RPC)' % (obj, method))
                cr.commit()
            except Exception:
                cr.rollback()
                raise
        finally:
            cr.close()
        return res

    def exec_workflow_cr(self, cr, uid, obj, method, *args):
        wf_service = netsvc.LocalService("workflow")
        return wf_service.trg_validate(uid, obj, args[0], method, cr)

    @check
    def exec_workflow(self, db, uid, obj, method, *args):
        cr = pooler.get_db(db).cursor()
        try:
            try:
                res = self.exec_workflow_cr(cr, uid, obj, method, *args)
                cr.commit()
            except Exception:
                cr.rollback()
                raise
        finally:
            cr.close()
        return res

    def obj_list(self):
        return self.obj_pool.keys()

    # adds a new object instance to the object pool.
    # if it already existed, the instance is replaced
    def add(self, name, obj_inst):
        if name in self.obj_pool:
            del self.obj_pool[name]
        self.obj_pool[name] = obj_inst

        module = str(obj_inst.__class__)[6:]
        module = module[:len(module)-1]
        module = module.split('.')[0][2:]
        self.module_object_list.setdefault(module, []).append(obj_inst)

    # Return None if object does not exist
    def get(self, name):
        obj = self.obj_pool.get(name, None)
        return obj

    #TODO: pass a list of modules to load
    def instanciate(self, module, cr):
        res = []
        class_list = module_class_list.get(module, [])
        for klass in class_list:
            res.append(klass.createInstance(self, module, cr))
        return res

    def set_debug(self, db, obj, do_debug=True):
        object = pooler.get_pool(db).get(obj)
        if not object:
            raise except_osv('Object Error', 'Object %s doesn\'t exist' % str(obj))
        try:
            object._debug = do_debug
            return True
        except:
            raise

class osv_base(object):
    def __init__(self, pool, cr):
        pool.add(self._name, self)
        self.pool = pool
        super(osv_base, self).__init__(cr)

    def __new__(cls):
        module = str(cls)[6:]
        module = module[:len(module)-1]
        module = module.split('.')[0][2:]
        if not hasattr(cls, '_module'):
            cls._module = module
        module_class_list.setdefault(cls._module, []).append(cls)
        class_pool[cls._name] = cls
        if module not in module_list:
            module_list.append(cls._module)
        return None

class osv_memory(osv_base, orm.orm_memory):
    #
    # Goal: try to apply inheritancy at the instanciation level and
    #       put objects in the pool var
    #
    def createInstance(cls, pool, module, cr):
        parent_names = getattr(cls, '_inherit', None)
        if parent_names:
            if isinstance(parent_names, (str, unicode)):
                name = cls._name or parent_names
                parent_names = [parent_names]
            else:
                name = cls._name
            if not name:
                raise TypeError('_name is mandatory in case of multiple inheritance')

            for parent_name in ((type(parent_names)==list) and parent_names or [parent_names]):
                parent_class = pool.get(parent_name).__class__
                assert pool.get(parent_name), "parent class %s does not exist in module %s !" % (parent_name, module)
                nattr = {}
                for s in ('_columns', '_defaults'):
                    new = copy.copy(getattr(pool.get(parent_name), s))
                    if hasattr(new, 'update'):
                        new.update(cls.__dict__.get(s, {}))
                    else:
                        new.extend(cls.__dict__.get(s, []))
                    nattr[s] = new
                cls = type(name, (cls, parent_class), nattr)

        obj = object.__new__(cls)
        obj.__init__(pool, cr)
        return obj
    createInstance = classmethod(createInstance)

class osv(osv_base, orm.orm):
    #
    # Goal: try to apply inheritancy at the instanciation level and
    #       put objects in the pool var
    #
    def createInstance(cls, pool, module, cr):
        parent_names = getattr(cls, '_inherit', None)
        if parent_names:
            if isinstance(parent_names, (str, unicode)):
                name = cls._name or parent_names
                parent_names = [parent_names]
            else:
                name = cls._name
            if not name:
                raise TypeError('_name is mandatory in case of multiple inheritance')

            for parent_name in ((type(parent_names)==list) and parent_names or [parent_names]):
                parent_class = pool.get(parent_name).__class__
                assert pool.get(parent_name), "parent class %s does not exist in module %s !" % (parent_name, module)
                nattr = {}
                for s in ('_columns', '_defaults', '_inherits', '_constraints', '_sql_constraints'):
                    new = copy.copy(getattr(pool.get(parent_name), s))
                    if hasattr(new, 'update'):
                        new.update(cls.__dict__.get(s, {}))
                    else:
                        if s=='_constraints':
                            for c in cls.__dict__.get(s, []):
                                exist = False
                                for c2 in range(len(new)):
                                     #For _constraints, we should check field and methods as well
                                     if new[c2][2]==c[2] and new[c2][0]==c[0]:
                                        new[c2] = c
                                        exist = True
                                        break
                                if not exist:
                                    new.append(c)
                        else:
                            new.extend(cls.__dict__.get(s, []))
                    nattr[s] = new
                cls = type(name, (cls, parent_class), nattr)
        obj = object.__new__(cls)
        obj.__init__(pool, cr)
        return obj
    createInstance = classmethod(createInstance)

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<|endoftext|>


<commit_before># Copyright 2010 Alex K (wtwf.com) All rights reserved.

# useful?
# http://www.opensearch.org/Specifications/OpenSearch/Extensions/Suggestions/1.0
# http://www.opensearch.org/Specifications/OpenSearch/Extensions/Suggestions/1.1
# https://developer.mozilla.org/en-US/docs/Supporting_search_suggestions_in_search_plugins
"""
suggets reply is
```
[
  "imdb star wa",
  [
    "imdb Star Wars on IMDb",
    "imdb Solo: A Star Wars Story",
  ],
  [
    "Star Wars on IMDb",
    "Solo: A Star Wars Story",
  ],
  [
    "https://imdb.com/title//star-wars",
    "https://imdb.com/title/tt3778644",
  ]
]
```
"""
__author__ = 'wtwf.com (Alex K)'

import collections
import json
import logging
import os
import re
import urllib

import google.appengine.api.urlfetch

import jinja2

from wtwf import wtwfhandler
import model
import snippy_config

class SuggestHandler(wtwfhandler.WtwfHandler):
  def get(self):
    q = self.request.get('q')
    parts = q.split(' ', 1)
    if len(parts) == 2:
      snippy = model.GetByKeyword(parts[0])
      if snippy and snippy.suggest_url:
        if snippy.private:
          self.AssertAllowed()
        parts[1] = parts[1].decode('utf-8')
        url = snippy.suggest_url
        if '%s' in url:
          url = snippy.suggest_url.replace('%s', urllib.quote(parts[1]))
        elif '{{' in url:
          url = str(jinja2.Template(url).render({
            'searchTerms': urllib.quote(parts[1])
          }))
        elif '{searchTerms}' in url:
          url = snippy.suggest_url.replace('{searchTerms}', urllib.quote(parts[1]))
        res = google.appengine.api.urlfetch.fetch(url)
        reply = fixupSuggestReply(url, parts, res.content)
        self.response.headers['Content-Type'] = 'application/x-suggestions+json'
        logging.info("REPLY\n%s", json.dumps(json.loads(reply), indent=2, sort_keys=True))
        self.response.out.write(reply)

class SuggestXmlHandler(wtwfhandler.WtwfHandler):
  def get(self):
    if os.getenv('SERVER_SOFTWARE', '').startswith('Google App Engine/'):
      DEV = ''
    else:
      DEV = '-dev'
    config = snippy_config.SnippyConfig()
    template_values = {
      'host': self.GetBaseUrl(),
      'shortName': config.get('shortName', 'shortName') + DEV,
      'description': config.get('description', 'description') + DEV,
      'developer': config.get('developer', 'developer') + DEV,
    }
    self.response.headers['Content-Type'] = 'application/opensearchdescription+xml'
    self.SendTemplate('opensearch.xml', template_values)

def fixupGoogles(keyword, reply_key, reply):
  """the google.com/s format."""
  obj = json.loads(getJson(reply))
  # the first element of arrays that are 4 deep
  choices = flatten(firstNdeep(obj, 4))
  choices = ['%s %s' % (keyword, x) for x in choices if isinstance(x, (str, unicode)) and x[-1] != '=']
  return json.dumps([reply_key, choices])

def fixupImdb(keyword, reply_key, reply):
  obj = json.loads(getJson(reply))
  prefixes = collections.defaultdict(lambda: 'title')
  prefixes.update({'tt': 'title', 'nm': 'name'})
  # I wish chrome understood the last 3 elements of this array.
  obj = [
    reply_key,
    ['%s %s' % (keyword, x[u'l']) for x in obj],
    [x[u'l'] for x in obj],
    ['https://imdb.com/%s/%s' % (prefixes[x[u'id'][0:2]], x[u'id']) for x in obj],
  ]
  return json.dumps(obj)

JSONP_START_RE = re.compile(r'^[a-zA-Z0-9_.]+\(', re.MULTILINE)
FIXUP_MAP = {
  r'^https://www\.google\.com/s\?tbm=': fixupGoogles,
  r'^https://sg\.media-imdb\.com/suggests': fixupImdb,
}

def getJson(s):
  try:
    s = s[s.find('['):s.rfind(']') + 1]
    if s.find('[\\"') != -1:
      s = s.decode('string_escape') # google.com/s does this. Sends back a json string as a key to a dict.
    return s
  except e:
    pass
  return s

def fixupSuggestReply(url, parts, reply):
  """Takes a suggest url reply and add the keyword to every suggestion."""
  keyword = parts[0]
  search_term = parts[1]
  reply_key = ' '.join(parts)

  for (regex, func) in FIXUP_MAP.items():
    if re.search(regex, url):
      return func(keyword, reply_key, reply)

  # make sure it's not jsonp
  reply = getJson(reply)

  # first decode it
  obj = json.loads(reply)

  if len(obj) >= 2 and isinstance(obj[1], list):
    # result is ['searchTerm', ['option1', 'option2']]
    # fix up the search term in the result
    obj[0] = reply_key
    obj[1] = [fixupSuggestReplyEntry(keyword, x) for x in obj[1]]
  else:
    # assume result is ['option1', 'option2']
    obj = [reply_key, [fixupSuggestReplyEntry(keyword, x) for x in obj]]

  # then reencode it and send it back
  return json.dumps(obj)

def fixupSuggestReplyEntry(keyword, entry):
  if isinstance(entry, basestring):
    return keyword + ' ' + removeBold(entry)
  else:
    return keyword + ' ' + removeBold(entry[0])

BOLD_RE = re.compile(r'(\u003C/?b\u003E|</?b>)', re.MULTILINE)

def removeBold(entry):
  return BOLD_RE.sub('', entry)

def flatten(i):
  if isinstance(i, list) or isinstance(i, tuple):
    for j in i:
      for x in flatten(j):
        yield x
  else:
    if i:
      yield i

def firstNdeep(arr, depth, currentDepth=0):
  if not isinstance(arr, list):
    return None

  if depth == currentDepth:
    return arr[0]
  return [firstNdeep(x, depth, currentDepth + 1) for x in arr]
<commit_msg>support for goodreads suggest<commit_after># Copyright 2010 Alex K (wtwf.com) All rights reserved.

# useful?
# http://www.opensearch.org/Specifications/OpenSearch/Extensions/Suggestions/1.0
# http://www.opensearch.org/Specifications/OpenSearch/Extensions/Suggestions/1.1
# https://developer.mozilla.org/en-US/docs/Supporting_search_suggestions_in_search_plugins
"""
suggets reply is
```
[
  "imdb star wa",
  [
    "imdb Star Wars on IMDb",
    "imdb Solo: A Star Wars Story",
  ],
  [
    "Star Wars on IMDb",
    "Solo: A Star Wars Story",
  ],
  [
    "https://imdb.com/title//star-wars",
    "https://imdb.com/title/tt3778644",
  ]
]
```
"""
__author__ = 'wtwf.com (Alex K)'

import collections
import json
import logging
import os
import re
import urllib

import google.appengine.api.urlfetch

import jinja2

from wtwf import wtwfhandler
import model
import snippy_config

class SuggestHandler(wtwfhandler.WtwfHandler):
  def get(self):
    q = self.request.get('q')
    parts = q.split(' ', 1)
    if len(parts) == 2:
      snippy = model.GetByKeyword(parts[0])
      if snippy and snippy.suggest_url:
        if snippy.private:
          self.AssertAllowed()
        parts[1] = parts[1].decode('utf-8')
        url = snippy.suggest_url
        if '%s' in url:
          url = snippy.suggest_url.replace('%s', urllib.quote(parts[1]))
        elif '{{' in url:
          url = str(jinja2.Template(url).render({
            'searchTerms': urllib.quote(parts[1])
          }))
        elif '{searchTerms}' in url:
          url = snippy.suggest_url.replace('{searchTerms}', urllib.quote(parts[1]))
        # logging.info('loading: %r', url)
        res = google.appengine.api.urlfetch.fetch(url)
        reply = fixupSuggestReply(url, parts, res.content)
        self.response.headers['Content-Type'] = 'application/x-suggestions+json'
        # logging.info("REPLY\n%s", json.dumps(json.loads(reply), indent=2, sort_keys=True))
        self.response.out.write(reply)

class SuggestXmlHandler(wtwfhandler.WtwfHandler):
  def get(self):
    if os.getenv('SERVER_SOFTWARE', '').startswith('Google App Engine/'):
      DEV = ''
    else:
      DEV = '-dev'
    config = snippy_config.SnippyConfig()
    template_values = {
      'host': self.GetBaseUrl(),
      'shortName': config.get('shortName', 'shortName') + DEV,
      'description': config.get('description', 'description') + DEV,
      'developer': config.get('developer', 'developer') + DEV,
    }
    self.response.headers['Content-Type'] = 'application/opensearchdescription+xml'
    self.SendTemplate('opensearch.xml', template_values)

def fixupGoogles(keyword, reply_key, reply):
  """the google.com/s format."""
  obj = getJson(reply)
  # the first element of arrays that are 4 deep
  choices = flatten(firstNdeep(obj, 4))
  choices = ['%s %s' % (keyword, x) for x in choices if isinstance(x, (str, unicode)) and x[-1] != '=']
  return json.dumps([reply_key, choices])

def fixupImdb(keyword, reply_key, reply):
  obj = getJson(reply)
  prefixes = collections.defaultdict(lambda: 'title')
  prefixes.update({'tt': 'title', 'nm': 'name'})
  # I wish chrome understood the last 3 elements of this array.
  obj = [
    reply_key,
    ['%s %s' % (keyword, x[u'l']) for x in obj],
    [x[u'l'] for x in obj],
    ['https://imdb.com/%s/%s' % (prefixes[x[u'id'][0:2]], x[u'id']) for x in obj],
  ]
  return json.dumps(obj)

def fixupGoodreads(keyword, reply_key, reply):
  obj = getJson(reply)
  obj = [
    reply_key,
    ['%s %s' % (keyword, x[u'title']) for x in obj],
    [x[u'title'] for x in obj],
    ['https://www.goodreads.com%s' % x[u'bookUrl'] for x in obj],
  ]
  return json.dumps(obj)

JSONP_START_RE = re.compile(r'^[a-zA-Z0-9_.]+\(', re.MULTILINE)
FIXUP_MAP = {
  r'^https://www\.google\.com/s\?tbm=': fixupGoogles,
  r'^https://sg\.media-imdb\.com/suggests': fixupImdb,
  r'^https://www.goodreads.com': fixupGoodreads,
}

def getJson(s):
  try:
    s = s[s.find('['):s.rfind(']') + 1]
    if s.find('[\\"') != -1:
      s = s.decode('string_escape') # google.com/s does this. Sends back a json string as a key to a dict.
  except e:
    pass
  return json.loads(s)

def fixupSuggestReply(url, parts, reply):
  """Takes a suggest url reply and add the keyword to every suggestion."""
  keyword = parts[0]
  search_term = parts[1]
  reply_key = ' '.join(parts)

  for (regex, func) in FIXUP_MAP.items():
    if re.search(regex, url):
      return func(keyword, reply_key, reply)

  # make sure it's not jsonp
  obj = getJson(reply)

  if len(obj) >= 2 and isinstance(obj[1], list):
    # result is ['searchTerm', ['option1', 'option2']]
    # fix up the search term in the result
    obj[0] = reply_key
    obj[1] = [fixupSuggestReplyEntry(keyword, x) for x in obj[1]]
  else:
    # assume result is ['option1', 'option2']
    obj = [reply_key, [fixupSuggestReplyEntry(keyword, x) for x in obj]]

  # then reencode it and send it back
  return json.dumps(obj)

def fixupSuggestReplyEntry(keyword, entry):
  if isinstance(entry, basestring):
    return keyword + ' ' + removeBold(entry)
  else:
    return keyword + ' ' + removeBold(entry[0])

BOLD_RE = re.compile(r'(\u003C/?b\u003E|</?b>)', re.MULTILINE)

def removeBold(entry):
  return BOLD_RE.sub('', entry)

def flatten(i):
  if isinstance(i, list) or isinstance(i, tuple):
    for j in i:
      for x in flatten(j):
        yield x
  else:
    if i:
      yield i

def firstNdeep(arr, depth, currentDepth=0):
  if not isinstance(arr, list):
    return None

  if depth == currentDepth:
    return arr[0]
  return [firstNdeep(x, depth, currentDepth + 1) for x in arr]
<|endoftext|>


<commit_before>"""
hellofriend.py
Author: Suhan
Credit: None

Assignment:

Write and submit an interactive Python program that asks for the user's name and age,
then prints how much older Python is than the user (based on a simple comparison of
birth year). Python's first public release occurred in 1991. Something like this:

Please tell me your name: Guido
Please tell me your age: 16
Hello, Guido. Python is 8 years older than you are!

Note that the text: "Guido" and "16" are entered by the user running the program.
The final line ("Hello...") is generated dynamically when you run the program, based
on the name and age that the user enters.
"""

name=input("Please tell me your name: ")
age= input("please<commit_msg>Updated from Brython Server: 9/9/2015 2:00:18 PM<commit_after>"""
hellofriend.py
Author: Suhan
Credit: None

Assignment:

Write and submit an interactive Python program that asks for the user's name and age,
then prints how much older Python is than the user (based on a simple comparison of
birth year). Python's first public release occurred in 1991. Something like this:

Please tell me your name: Guido
Please tell me your age: 16
Hello, Guido. Python is 8 years older than you are!

Note that the text: "Guido" and "16" are entered by the user running the program.
The final line ("Hello...") is generated dynamically when you run the program, based
on the name and age that the user enters.
"""

name=input("Please tell me your name: ")
age= input("Please tell me your age: ")
s="{0} heeellooo {1}"
print(s. format(name), 24-int(age))<|endoftext|>


<commit_before># :coding: utf-8
# :copyright: Copyright (c) 2013 Martin Pengelly-Phillips
# :license: See LICENSE.txt.

import sys

def main(arguments=None):
    '''Segue command line interface.'''

if __name__ == '__main__':
    '''Execute command line interface.'''
    if '__main__.py' in sys.argv[0]:
        sys.argv[0] = 'segue'

    raise SystemExit(int(main() is False))

<commit_msg>[#7] Add command line interface to optimise package for Houdini.<commit_after># :coding: utf-8
# :copyright: Copyright (c) 2013 Martin Pengelly-Phillips
# :license: See LICENSE.txt.

import os
import sys
import argparse
import subprocess
import errno

try:
    import json
except ImportError:
    import simplejson as json

def main(arguments=None):
    '''Segue command line interface.'''
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers()

    optimise_parser = subparsers.add_parser(
        'optimise-for', help='Optimise package.'
    )
    optimise_parser.add_argument('application', choices=['houdini'],
                                 help='Application to optimise package for.')
    optimise_parser.add_argument('package', type=_valid_package,
                                 help='Path to package file to optimise.')
    optimise_parser.set_defaults(func=optimise_for)

    namespace = parser.parse_args(arguments)
    namespace.func(namespace)

def optimise_for(namespace):
    '''Handle subcommand.'''
    if namespace.application == 'houdini':
        optimise_for_houdini(namespace.package)
        return

    raise ValueError(
        'Cannot optimise for unsupported package: {0}'
        .format(namespace.application)
    )

def optimise_for_houdini(package_path):
    '''Optimise *package* for use in Houdini by adding bgeo format.'''
    with open(package_path, 'r') as package_file:
        package = json.load(package_file)

    if 'houdini' in package:
        print 'Package already optimised for Houdini.'
        return

    start = package['start']
    stop = package['stop']

    root_path = os.path.dirname(package_path)
    bgeo_path = os.path.join(root_path, 'bgeo', '$FF.bgeo')

    try:
        os.makedirs(os.path.dirname(bgeo_path))
    except OSError as error:
        if error.errno != errno.EEXIST:
            raise

    if not root_path.endswith(os.sep):
        root_path += os.sep

    bgeo_relative_path = bgeo_path[len(root_path):].replace(os.sep, '/')

    command = [
        'hython', '-c',
        (
        'import os;'
        'import segue.backend.host.houdini;'
        'import segue;'
        'host = segue.backend.host.houdini.HoudiniHost();'
        'node = host.load_package("{0}");'
        'implementation_node = node.node("output/implementation");'
        'rop_node = implementation_node.createOutputNode("rop_geometry");'
        'rop_node.parm("sopoutput").set("{1}");'
        'rop_node.parm("trange").set(1);'
        'rop_node.parm("f1").set({2});'
        'rop_node.parm("f2").set({3});'
        'rop_node.render();'
        .format(
            package_path.replace('\\', '/'),
            bgeo_path.replace('\\', '/'),
            start, stop)
        )
    ]
    subprocess.check_call(command)

    package['houdini'] = bgeo_relative_path
    with open(package_path, 'w') as package_file:
        json.dump(package, package_file)

    print 'Optimisation succeeded.'

def _valid_package(value):
    '''Check *value* refers to a valid package.'''
    if value.endswith('package.json') and os.path.isfile(value):
        return value

    raise argparse.ArgumentTypeError(
        '{0!r} is not a valid path to a package'.format(value)
    )

# Execute command line interface.
if __name__ == '__main__':
    if '__main__.py' in sys.argv[0]:
        sys.argv[0] = 'segue'

    main(sys.argv[1:])
    raise SystemExit(0)

<|endoftext|>


<commit_before>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

import re
import time
import socket

from elasticsearch import Elasticsearch
from neutronclient.common.exceptions import NeutronClientException

from cosmo_tester.framework.testenv import TestCase
from cosmo_tester.framework.handlers.openstack import OpenstackHandler

ELASTICSEARCH_PORT = 9200

TIMESTAMP_PATTERN = '\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}'

DEFAULT_EXECUTE_TIMEOUT = 1800

class ElasticsearchTimestampFormatTest(TestCase):

    """
    CFY-54
    this test checks Elasticsearch Timestamp Format.
    it creates events by uploading a blueprint and creating deployment.
    after creating events the test connects to Elasticsearch and compares
    Timestamp Format of the events to a regular expression.

    This test requires access to the management on port 9200 (elastic search",
    The rule is added by _create_elasticsearch_rule
    """
    def _create_elasticsearch_rule(self):
        os_handler = OpenstackHandler(self.env)
        neutron_client = os_handler.openstack_clients()[1]
        sgr = {
            'direction': 'ingress',
            'ethertype': 'IPv4',
            'port_range_max': str(ELASTICSEARCH_PORT),
            'port_range_min': str(ELASTICSEARCH_PORT),
            'protocol': 'tcp',
            'remote_group_id': None,
            'remote_ip_prefix': '0.0.0.0/0',
            }

        mng_sec_grp_name = self.env.management_security_group

        mng_sec_grp = neutron_client. \
            list_security_groups(name=mng_sec_grp_name)['security_groups'][0]

        sg_id = mng_sec_grp['id']
        sgr['security_group_id'] = sg_id
        try:
            self.elasticsearch_rule = neutron_client.\
                create_security_group_rule(
                    {'security_group_rule': sgr})['security_group_rule']['id']
            if not self._wait_for_open_port(self.env.management_ip,
                                            ELASTICSEARCH_PORT,
                                            60):
                raise Exception('Couldn\'t open elasticsearch port')

        except NeutronClientException as e:
            self.elasticsearch_rule = None
            self.logger.warning("Got NeutronClientException({0}). Resuming"
                                .format(e))
            pass

    def setUp(self):
        super(ElasticsearchTimestampFormatTest, self).setUp()
        self._create_elasticsearch_rule()

    def _delete_elasticsearch_rule(self):
        if self.elasticsearch_rule is not None:
            os_handler = OpenstackHandler(self.env)
            neutron_client = os_handler.openstack_clients()[1]
            neutron_client.delete_security_group_rule(self.elasticsearch_rule)

    def tearDown(self):
        self.execute_uninstall()
        self._delete_elasticsearch_rule()
        super(ElasticsearchTimestampFormatTest, self).tearDown()

    def _check_port(self, ip, port):
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        result = sock.connect_ex((ip, port))
        return result == 0

    def _wait_for_open_port(self, ip, port, timeout):
        timeout = time.time() + timeout
        is_open = False
        while not is_open:
            if time.time() > timeout:
                break
            is_open = self._check_port(ip, port)
        return is_open

    def test_events_timestamp_format(self):
        self.blueprint_path = self.copy_blueprint('mocks')
        self.blueprint_yaml = self.blueprint_path / 'empty-bp.yaml'

        deployment_id = self.test_id
        self.upload_deploy_and_execute_install(deployment_id=deployment_id)

        #  connect to Elastic search

        es = Elasticsearch(self.env.management_ip +
                           ':' + str(ELASTICSEARCH_PORT))

        res = es.search(index="cloudify_events",
                        body={"query": {"match":
                                        {"deployment_id": deployment_id}}})
        self.logger.info("Got %d Hits:" % res['hits']['total'])
        #  check if events were created
        self.assertNotEqual(0, res['hits']['total'],
                            'There are no events in for '
                            'deployment ' + deployment_id)

        #  loop over all the events and compare timestamp to regular expression
        for hit in res['hits']['hits']:
            if not (re.match(TIMESTAMP_PATTERN, hit["_source"]['timestamp'])):
                self.fail('Got {0}. Does not match format '
                          'YYYY-MM-DD HH:MM:SS.***'
                          .format(hit["_source"]['timestamp']))
<commit_msg>Added sleep when waiting for open port<commit_after>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

import re
import time
import socket

from elasticsearch import Elasticsearch
from neutronclient.common.exceptions import NeutronClientException

from cosmo_tester.framework.testenv import TestCase
from cosmo_tester.framework.handlers.openstack import OpenstackHandler

ELASTICSEARCH_PORT = 9200

TIMESTAMP_PATTERN = '\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}'

DEFAULT_EXECUTE_TIMEOUT = 1800

class ElasticsearchTimestampFormatTest(TestCase):
    """
    CFY-54
    this test checks Elasticsearch Timestamp Format.
    it creates events by uploading a blueprint and creating deployment.
    after creating events the test connects to Elasticsearch and compares
    Timestamp Format of the events to a regular expression.

    This test requires access to the management on port 9200 (elastic search",
    The rule is added by _create_elasticsearch_rule
    """

    def _create_elasticsearch_rule(self):
        os_handler = OpenstackHandler(self.env)
        neutron_client = os_handler.openstack_clients()[1]
        sgr = {
            'direction': 'ingress',
            'ethertype': 'IPv4',
            'port_range_max': str(ELASTICSEARCH_PORT),
            'port_range_min': str(ELASTICSEARCH_PORT),
            'protocol': 'tcp',
            'remote_group_id': None,
            'remote_ip_prefix': '0.0.0.0/0',
        }

        mng_sec_grp_name = self.env.management_security_group

        mng_sec_grp = neutron_client. \
            list_security_groups(name=mng_sec_grp_name)['security_groups'][0]

        sg_id = mng_sec_grp['id']
        sgr['security_group_id'] = sg_id
        try:
            self.elasticsearch_rule = neutron_client. \
                create_security_group_rule(
                    {'security_group_rule': sgr})['security_group_rule']['id']
            if not self._wait_for_open_port(self.env.management_ip,
                                            ELASTICSEARCH_PORT,
                                            60):
                raise Exception('Couldn\'t open elasticsearch port')

        except NeutronClientException as e:
            self.elasticsearch_rule = None
            self.logger.warning("Got NeutronClientException({0}). Resuming"
                                .format(e))
            pass

    def setUp(self):
        super(ElasticsearchTimestampFormatTest, self).setUp()
        self._create_elasticsearch_rule()

    def _delete_elasticsearch_rule(self):
        if self.elasticsearch_rule is not None:
            os_handler = OpenstackHandler(self.env)
            neutron_client = os_handler.openstack_clients()[1]
            neutron_client.delete_security_group_rule(self.elasticsearch_rule)

    def tearDown(self):
        self.execute_uninstall()
        self._delete_elasticsearch_rule()
        super(ElasticsearchTimestampFormatTest, self).tearDown()

    def _check_port(self, ip, port):
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        result = sock.connect_ex((ip, port))
        return result == 0

    def _wait_for_open_port(self, ip, port, timeout):
        timeout = time.time() + timeout
        is_open = False
        while not is_open:
            if time.time() > timeout:
                break
            time.sleep(1)
            is_open = self._check_port(ip, port)
        return is_open

    def test_events_timestamp_format(self):
        self.blueprint_path = self.copy_blueprint('mocks')
        self.blueprint_yaml = self.blueprint_path / 'empty-bp.yaml'

        deployment_id = self.test_id
        self.upload_deploy_and_execute_install(deployment_id=deployment_id)

        #  connect to Elastic search

        es = Elasticsearch(self.env.management_ip +
                           ':' + str(ELASTICSEARCH_PORT))

        res = es.search(index="cloudify_events",
                        body={"query": {"match":
                                        {"deployment_id": deployment_id}}})
        self.logger.info("Got %d Hits:" % res['hits']['total'])
        #  check if events were created
        self.assertNotEqual(0, res['hits']['total'],
                            'There are no events in for '
                            'deployment ' + deployment_id)

        #  loop over all the events and compare timestamp to regular expression
        for hit in res['hits']['hits']:
            if not (re.match(TIMESTAMP_PATTERN, hit["_source"]['timestamp'])):
                self.fail('Got {0}. Does not match format '
                          'YYYY-MM-DD HH:MM:SS.***'
                          .format(hit["_source"]['timestamp']))
<|endoftext|>


<commit_before>"""JTS type casting. Patterned on okfn/messy-tables"""
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from . import compat
from dateutil.parser import parse as date_parse

class JTSType(object):

    py = None

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        if value in ('', None):
            return False

class StringType(JTSType):

    py = compat.str

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        if value is None:
            return False
        if isinstance(value, self.py):
            return True
        return compat.str(value)

class IntegerType(JTSType):

    py = int

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        super(IntegerType, self).cast(value)
        if isinstance(value, self.py):
            return True
        try:
            return self.py(value)
        except ValueError:
            return False

class NumberType(JTSType):

    py = float, decimal.Decimal
    seperators = ',;'
    currencies = '$'

    def __init__(self, strip_seperators=False, strip_currencies=False):
        self.strip_seperators = strip_seperators
        self.strip_currencies = strip_currencies

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        super(NumberType, self).cast(value)
        if self.strip_seperators:
            value = re.sub('[{0}]'.format(self.seperators), '', value)
        if self.strip_currency:
             value = re.sub('[{0}]'.format(self.currencies), '', value)
        if isinstance(value, self.py):
            return True
        try:
            return decimal.Decimal(value)
        except decimal.InvalidOperation:
            return False

class DateType(JTSType):

    py = datetime.date
    format = '%Y-%m-%d'

    def __init__(self, strict=False):
        self.strict = strict

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        super(DateType, self).cast(value)
        try:
            if self.strict:
                return datetime.datetime.strptime(value, self.format).date()
            else:
                return date_parse(value).date()
        except ValueError:
            return False

class TimeType(JTSType):

    py = datetime.time
    format = '%H:%M:%S'

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        super(TimeType, self).cast(value)
        try:
            return time.strptime(value, self.format)
        except ValueError:
            return False

class DateTimeType(JTSType):
    py = datetime.datetime
    format = '%Y-%m-%dT%H:%M:%SZ'

    def __init__(self, strict=False):
        self.strict = strict

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        super(DateTimeType, self).cast(value)
        try:
            if self.strict:
                return datetime.datetime.strptime(value, self.format)
            else:
                return date_parse(value)
        except ValueError:
            return False

class BooleanType(JTSType):

    py = bool
    true_values = ('yes', 'y', 'true', '0')
    false_values = ('no', 'n','false', '1')

    def __init__(self, true_values=None, false_values=None):
        if true_values is not None:
            self.true_values = true_values
        if false_values is not None:
            self.false_values = false_values

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        super(BooleanType, self).cast(value)
        value = value.strip().lower()
        if value in self.true_values:
            return True
        if value in self.false_values:
            return False
        return False

class ArrayType(JTSType):

    py = list

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        super(ArrayType, self).cast(value)
        if isinstance(value, self.py):
            return True
        try:
            value = json.loads(value)
            if isinstance(value, self.py):
                return True
            else:
                return False
        except ValueError:
            return False

class ObjectType(JTSType):

    py = dict

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        super(ObjectType, self).cast(value)
        if isinstance(value, self.py):
            return True
        try:
            value = json.loads(value)
            if isinstance(value, self.py):
                return True
            else:
                return False
        except ValueError:
            return False

class AnyType(JTSType):

    def cast(self, value):
        return True
<commit_msg>Fix missing imports<commit_after>"""JTS type casting. Patterned on okfn/messy-tables"""
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import re
import decimal
import datetime
import time
import json
from . import compat
from dateutil.parser import parse as date_parse

class JTSType(object):

    py = None

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        if value in ('', None):
            return False

class StringType(JTSType):

    py = compat.str

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        if value is None:
            return False
        if isinstance(value, self.py):
            return True
        return compat.str(value)

class IntegerType(JTSType):

    py = int

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        super(IntegerType, self).cast(value)
        if isinstance(value, self.py):
            return True
        try:
            return self.py(value)
        except ValueError:
            return False

class NumberType(JTSType):

    py = float, decimal.Decimal
    seperators = ',;'
    currencies = '$'

    def __init__(self, strip_seperators=False, strip_currencies=False):
        self.strip_seperators = strip_seperators
        self.strip_currencies = strip_currencies

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        super(NumberType, self).cast(value)
        if self.strip_seperators:
            value = re.sub('[{0}]'.format(self.seperators), '', value)
        if self.strip_currency:
             value = re.sub('[{0}]'.format(self.currencies), '', value)
        if isinstance(value, self.py):
            return True
        try:
            return decimal.Decimal(value)
        except decimal.InvalidOperation:
            return False

class DateType(JTSType):

    py = datetime.date
    format = '%Y-%m-%d'

    def __init__(self, strict=False):
        self.strict = strict

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        super(DateType, self).cast(value)
        try:
            if self.strict:
                return datetime.datetime.strptime(value, self.format).date()
            else:
                return date_parse(value).date()
        except ValueError:
            return False

class TimeType(JTSType):

    py = time
    format = '%H:%M:%S'

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        super(TimeType, self).cast(value)
        try:
            return time.strptime(value, self.format)
        except ValueError:
            return False

class DateTimeType(JTSType):
    py = datetime.datetime
    format = '%Y-%m-%dT%H:%M:%SZ'

    def __init__(self, strict=False):
        self.strict = strict

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        super(DateTimeType, self).cast(value)
        try:
            if self.strict:
                return datetime.datetime.strptime(value, self.format)
            else:
                return date_parse(value)
        except ValueError:
            return False

class BooleanType(JTSType):

    py = bool
    true_values = ('yes', 'y', 'true', '0')
    false_values = ('no', 'n','false', '1')

    def __init__(self, true_values=None, false_values=None):
        if true_values is not None:
            self.true_values = true_values
        if false_values is not None:
            self.false_values = false_values

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        super(BooleanType, self).cast(value)
        value = value.strip().lower()
        if value in self.true_values:
            return True
        if value in self.false_values:
            return False
        return False

class ArrayType(JTSType):

    py = list

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        super(ArrayType, self).cast(value)
        if isinstance(value, self.py):
            return True
        try:
            value = json.loads(value)
            if isinstance(value, self.py):
                return True
            else:
                return False
        except ValueError:
            return False

class ObjectType(JTSType):

    py = dict

    def cast(self, value):
        """Return boolean if `value` can be cast as type `self.py`"""
        super(ObjectType, self).cast(value)
        if isinstance(value, self.py):
            return True
        try:
            value = json.loads(value)
            if isinstance(value, self.py):
                return True
            else:
                return False
        except ValueError:
            return False

class AnyType(JTSType):

    def cast(self, value):
        return True
<|endoftext|>