<commit_before># Copyright (c) 2007, Neville-Neil Consulting
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# Neither the name of Neville-Neil Consulting nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# File: $Id: $
#
# Author: George V. Neville-Neil
#
# Description: This module contains all the code which implementes the
# CLI for the Packet Debugger

import sys
import cmd  # Used to implement our CLI
import packetstream

class Command(cmd.Cmd):
    """The top level  Command Line Interpreter for the packet debugger.

    This class uses the built in command interpreter from Python to
    build up the full set of commands implemented by the debugger.
    Each command is written as a do_xxx function in this class

    Each command takes a string which should be parsed as arguments.

    Help follows the command."""

    def __init__(self, current, options, streams):
        self.current = current
        self.options = options
        self.streams = streams
        self.prompt = "pdb> "
        cmd.Cmd.__init__(self)

    def do_quit(self, message):
        print "Bye"
        sys.exit()

    def help_quit(self):
        print "Quit the debugger"

    def do_create(self, args):
        """Create a new stream from a file or network interface.

        Acceptable argument lists are:
        file filename
        interface interfacename
        """
        try:
            arg_list = args.split()
        except:
            self.help_create()
            return
        if (len(arg_list) != 2):
            self.help_create()
            return

        self.current = packetstream.Stream(self.options, arg_list[1])
        self.streams.append(self.current)

    def help_create(self):
        print "create file|interface filename|interface name"
        print "Create a new stream from a file or a network interface."

    def do_delete(self, args):
        """Delete a stream.  If N is present delete that stream."""
        if ((self.current == None) or (len(self.streams) <= 0)):
            print "No streams.  Use the create command to create one."
            return
        if (args == ""):
            self.streams.remove(self.current)
            if (len(self.streams) > 0):
                self.current = self.streams[0]
            else:
                self.current = None
        else:
            index = self.numarg(args)
            if (index == None):
                help_delete()
                return
            else:
                if (self.streams[index] == self.current):
                    self.current = None
                del self.streams[index]
                if (len(self.streams) > 0):
                    self.current = self.streams[0]

    def help_delete(self):
        print "delete (N)"
        print "Delete a Stream.  If N is given delete a specific stream otherwise delete the current one."

    def do_run(self, args):
        """Run the current stream or the one given in the index."""
        if (args == ""):
            if (self.current != None):
                self.current.run(0)
            else:
                print "No current stream.  Use the create or set commands"
        else:
            numarg = self.numarg(args)
            if (numarg == None):
                self.help_run()
                return
            if ((numarg < 0) or (numarg >= len(self.streams))):
                print "Stream must be between 0 and %d." % (len(self.streams) - 1)
                self.help_run(0)
                return
            self.streams[numarg].run(0)

    def help_run(self):
        print "run (N)"
        print "run the current stream or the stream given by the index"

    def complete_run(self, text, line, begidx, endidx):
        stream_list = []
        for i in range(0, len(self.streams)):
            stream_list.append(str(i))
        return stream_list

    def do_continue(self, args):
        """Continue the current stream or the one given in the index."""
        if (args == ""):
            if (self.current != None):
                self.current.next()
                self.current.run(self.current.position)
            else:
                print "No current stream.  Use the create or set commands"
        else:
            numarg = self.numarg(args)
            if (numarg == None):
                self.help_run()
                return
            if ((numarg < 0) or (numarg >= len(self.streams))):
                print "Stream must be between 0 and %d." % (len(self.streams) - 1)
                self.help_run()
                return
            self.streams[numarg].next()
            self.streams[numarg].run(self.streams[numarg].position)

    def help_continue(self):
        print "continue (N)"
        print "continue the current stream or the stream given by the index"

    def do_list(self, args):
        if (args == ""):
            if (self.current != None):
                self.current.list()
            else:
                print "No current stream.  Use the create or set commands"
        else:
            numarg = self.numarg(args)
            if (numarg == None):
                self.help_list()
                return
            if ((numarg < 0) or (numarg >= len(self.streams))):
                print "Stream must be between 0 and %d." % (len(self.streams) - 1)
                self.help_list()
                return
            self.streams[numarg].list()

    def help_list(self):
        print "list (N)"
        print "list packets from current stream or the stream at index N "

    def complete_list(self, text, line, begidx, endidx):
        stream_list = []
        for i in range(0, len(self.streams)):
            stream_list.append(str(i))
        return stream_list

    def do_print(self, args):
        if (args == ""):
            if (self.current == None):
                print "No current stream.  Use the create or set commands"
                return
            else:
                numarg = self.current.position
        else:
            numarg = self.numarg(args)
            if (numarg == None):
                self.help_print()
                return
        if ((numarg < 0) or (numarg > (len(self.current.packets) - 1))):
            print "Index must be between 0 and %d." % (len(self.current.packets) - 1)
            return

        self.current.display(numarg, self.current.packets[numarg])

    def help_print(self):
        print "print (N)"
        print "print the current packet in the current stream or packet N in the current stream"

    def do_send(self, args):
        "send the current packet or the packet given by N in the current stream"
        if (args == ""):
            if (self.current == None):
                print "No current stream.  Use the create or set commands"
                return
            else:
                numarg = self.current.position
        else:
            numarg = self.numarg(args)
            if (numarg == None):
                self.help_print()
                return
        if ((numarg < 0) or (numarg > (len(self.current.packets) - 1))):
            print "Index must be between 0 and %d." % (len(self.current.packets) - 1)
            return

        self.current.send(self.current.packets[numarg])

    def help_send(self):
        print "send (N)"
        print "send the current packet or the packet given by N in the current stream"

    def do_set(self, args):
        if len(args) <= 0:
            self.help_set()
            return
        try:
            (key, value) = args.split()
        except:
            self.help_set()
            return

        if key == "list_length":
            self.options.list_length = int(value)
        elif key == "layer":
            value = int(value)
            if value < 0:
                self.options.layer = None
            if value > 7:
                self.help_set()
            else:
                self.options.layer = value
        elif key == "current":
            if (value == None):
                print "Must supply a number between 0 and %d" % len(self.streams - 1)
                self.help_set()
                return
            value = int(value)
            if value < 0:
                print "Must be greater than 0"
                self.help_set()
                return
            if value >= len(self.streams):
                print "Must be less than %d" % (len(self.streams) - 1)
                self.help_set()
                return
            else:
                self.current = self.streams[value]

    def complete_set(self, text, line, begidx, endidx):
        """Completion for the set command"""
        if (text in ["li", "lis", "list", "list_", "list_l", "list_le", "list_len", "list_leng", "list_lengt"]):
            return ["list_length"]
        elif (text in ["la", "lay", "laye"]):
            return ["layer"]
        elif (text in ["c", "cu", "cur", "curr", "curre", "curren"]):
            return ["current"]
        else:
            if (line == "set current "):
                stream_list = []
                for i in range(0, len(self.streams)):
                    stream_list.append(str(i))
                return stream_list

    def help_set(self):
        print "set option value\n\noptions: list_length - how many packets to list at one time.\n         layer - ISO layer to show, -1 shows all\n         current - set the current stream we're inspecting [0..n]"

    def do_show(self, args):
        """Show the current options that are set."""
        print self.options

    def do_break(self, args):
        """set a breakpoint at the current index in the stream or at the numeric index given"""
        if (args == ""):
            numarg = self.current.position
        else:
            numarg = self.numarg(args)
        if (numarg == None):
            self.help_break()
            return
        self.current.add_break(numarg)

    def help_break(self):
        print "break (N)"
        print "set a breakpoint at the current index in the stream or at the numeric index given"

    def do_next(self, args):
        """Move to the next packet"""
        if (self.current == None):
            print "No current stream.  Use the create or set commands"
            return
        if (len(args) > 0):
            jump = self.numarg(args)
            if (jump == None):
                self.help_next()
                return
            else:
                self.current.next(jump)
        else:
            self.current.next()

    def help_next(self):
        print "next (N)"
        print "Move to the next packet in the list.  With a numeric argument, N, move N packets ahead in the list."

    def do_prev(self, args):
        """Move to the previous packet"""
        if (self.current == None):
            print "No current stream.  Use the create or set commands"
            return
        if (len(args) > 0):
            jump = self.numarg(args)
            if (jump == None):
                self.help_prev()
                return
            else:
                self.current.prev(jump)
        else:
            self.current.prev()

    def help_prev(self):
        print "prev (N)"
        print "Move to the previous packet in the list.  With a numeric argument, N, move N packets back in the list."

    def do_info(self, args):
        if (len (args) <= 0):
            if (self.current == None):
                print "No current stream.  Use the create or set commands"
                return
            print "Stream %d" % self.streams.index(self.current)
            print "---------"
            print self.current
        else:
            arg_list = args.split()
            if (arg_list[0] == "all"):
                for stream in self.streams:
                    print "Stream %d" % self.streams.index(stream)
                    print "---------"
                    print stream
            elif (arg_list[0] == "break"):
                print "Stream %d breakpoints %s" % (self.streams.index(self.current), self.current.breakpoints)
            elif (arg_list[0].isdigit() == True):
                index = int(arg_list[0])
                try:
                    stream = self.streams[index]
                except:
                    print "No stream %d" % index
                    return
                print "Stream %d" % index
                print "---------"
                print self.streams[index]
            else:
                self.help_info()
                return

    def complete_info(self, text, line, begidx, endidx):
        """Completion of the arguments for the info command"""
        if (text in ["a" , "al", "all"]):
            return ["all"]
        elif (text in ["b", "br", "bre", "brea"]):
            return ["break"]
        else:
            stream_list = []
            for i in range(0, len(self.streams)):
                stream_list.append(str(i))
            return stream_list

    def help_info(self):
        print "info [N | all]"
        print "Print out all the information on the current stream, a specific stream (N), or all streams."

    def numarg(self, args):
        """If the argument passed is numeric pass back the number, otherwise pass back None"""
        if (type(args) != str):
            print "N must be a number"
            return None
        if (args.isdigit() != True):
            print "N must be a number"
            return None
        return int(args)

<commit_msg>Rearrange the CLI code to group the commands and methods more cleanly.<commit_after># Copyright (c) 2007, Neville-Neil Consulting
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# Neither the name of Neville-Neil Consulting nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# File: $Id: $
#
# Author: George V. Neville-Neil
#
# Description: This module contains all the code which implementes the
# CLI for the Packet Debugger

import sys
import cmd  # Used to implement our CLI
import packetstream

class Command(cmd.Cmd):
    """The top level  Command Line Interpreter for the packet debugger.

    This class uses the built in command interpreter from Python to
    build up the full set of commands implemented by the debugger.
    Each command is written as a do_xxx function in this class

    Each command takes a string which should be parsed as arguments.

    Help follows the command."""

    def __init__(self, current, options, streams):
        self.current = current
        self.options = options
        self.streams = streams
        self.prompt = "pdb> "
        cmd.Cmd.__init__(self)

    def do_quit(self, message):
        print "Bye"
        sys.exit()

    def help_quit(self):
        print "Quit the debugger"

        #
        # Stream Management Commands
        #

    def do_create(self, args):
        """Create a new stream from a file or network interface.

        Acceptable argument lists are:
        file filename
        interface interfacename
        """
        try:
            arg_list = args.split()
        except:
            self.help_create()
            return
        if (len(arg_list) != 2):
            self.help_create()
            return

        self.current = packetstream.Stream(self.options, arg_list[1])
        self.streams.append(self.current)

    def help_create(self):
        print "create file|interface filename|interface name"
        print "Create a new stream from a file or a network interface."

    def do_delete(self, args):
        """Delete a stream.  If N is present delete that stream."""
        if ((self.current == None) or (len(self.streams) <= 0)):
            print "No streams.  Use the create command to create one."
            return
        if (args == ""):
            self.streams.remove(self.current)
            if (len(self.streams) > 0):
                self.current = self.streams[0]
            else:
                self.current = None
        else:
            index = self.numarg(args)
            if (index == None):
                help_delete()
                return
            else:
                if (self.streams[index] == self.current):
                    self.current = None
                del self.streams[index]
                if (len(self.streams) > 0):
                    self.current = self.streams[0]

    def help_delete(self):
        print "delete (N)"
        print "Delete a Stream.  If N is given delete a specific stream otherwise delete the current one."

    def do_run(self, args):
        """Run the current stream or the one given in the index."""
        if (args == ""):
            if (self.current != None):
                self.current.run(0)
            else:
                print "No current stream.  Use the create or set commands"
        else:
            numarg = self.numarg(args)
            if (numarg == None):
                self.help_run()
                return
            if ((numarg < 0) or (numarg >= len(self.streams))):
                print "Stream must be between 0 and %d." % (len(self.streams) - 1)
                self.help_run(0)
                return
            self.streams[numarg].run(0)

    def help_run(self):
        print "run (N)"
        print "run the current stream or the stream given by the index"

    def complete_run(self, text, line, begidx, endidx):
        stream_list = []
        for i in range(0, len(self.streams)):
            stream_list.append(str(i))
        return stream_list

    def do_continue(self, args):
        """Continue the current stream or the one given in the index."""
        if (args == ""):
            if (self.current != None):
                self.current.next()
                self.current.run(self.current.position)
            else:
                print "No current stream.  Use the create or set commands"
        else:
            numarg = self.numarg(args)
            if (numarg == None):
                self.help_run()
                return
            if ((numarg < 0) or (numarg >= len(self.streams))):
                print "Stream must be between 0 and %d." % (len(self.streams) - 1)
                self.help_run()
                return
            self.streams[numarg].next()
            self.streams[numarg].run(self.streams[numarg].position)

    def help_continue(self):
        print "continue (N)"
        print "continue the current stream or the stream given by the index"

    def do_info(self, args):
        if (len (args) <= 0):
            if (self.current == None):
                print "No current stream.  Use the create or set commands"
                return
            print "Stream %d" % self.streams.index(self.current)
            print "---------"
            print self.current
        else:
            arg_list = args.split()
            if (arg_list[0] == "all"):
                for stream in self.streams:
                    print "Stream %d" % self.streams.index(stream)
                    print "---------"
                    print stream
            elif (arg_list[0] == "break"):
                print "Stream %d breakpoints %s" % (self.streams.index(self.current), self.current.breakpoints)
            elif (arg_list[0].isdigit() == True):
                index = int(arg_list[0])
                try:
                    stream = self.streams[index]
                except:
                    print "No stream %d" % index
                    return
                print "Stream %d" % index
                print "---------"
                print self.streams[index]
            else:
                self.help_info()
                return

    def complete_info(self, text, line, begidx, endidx):
        """Completion of the arguments for the info command"""
        if (text in ["a" , "al", "all"]):
            return ["all"]
        elif (text in ["b", "br", "bre", "brea"]):
            return ["break"]
        else:
            stream_list = []
            for i in range(0, len(self.streams)):
                stream_list.append(str(i))
            return stream_list

    def help_info(self):
        print "info [N | all]"
        print "Print out all the information on the current stream, a specific stream (N), or all streams."

        #
        # Stream and Packet Inspection
        #

    def do_list(self, args):
        if (args == ""):
            if (self.current != None):
                self.current.list()
            else:
                print "No current stream.  Use the create or set commands"
        else:
            numarg = self.numarg(args)
            if (numarg == None):
                self.help_list()
                return
            if ((numarg < 0) or (numarg >= len(self.streams))):
                print "Stream must be between 0 and %d." % (len(self.streams) - 1)
                self.help_list()
                return
            self.streams[numarg].list()

    def help_list(self):
        print "list (N)"
        print "list packets from current stream or the stream at index N "

    def complete_list(self, text, line, begidx, endidx):
        stream_list = []
        for i in range(0, len(self.streams)):
            stream_list.append(str(i))
        return stream_list

    def do_print(self, args):
        if (args == ""):
            if (self.current == None):
                print "No current stream.  Use the create or set commands"
                return
            else:
                numarg = self.current.position
        else:
            numarg = self.numarg(args)
            if (numarg == None):
                self.help_print()
                return
        if ((numarg < 0) or (numarg > (len(self.current.packets) - 1))):
            print "Index must be between 0 and %d." % (len(self.current.packets) - 1)
            return

        self.current.display(numarg, self.current.packets[numarg])

    def help_print(self):
        print "print (N)"
        print "print the current packet in the current stream or packet N in the current stream"

    def do_send(self, args):
        "send the current packet or the packet given by N in the current stream"
        if (args == ""):
            if (self.current == None):
                print "No current stream.  Use the create or set commands"
                return
            else:
                numarg = self.current.position
        else:
            numarg = self.numarg(args)
            if (numarg == None):
                self.help_print()
                return
        if ((numarg < 0) or (numarg > (len(self.current.packets) - 1))):
            print "Index must be between 0 and %d." % (len(self.current.packets) - 1)
            return

        self.current.send(self.current.packets[numarg])

    def help_send(self):
        print "send (N)"
        print "send the current packet or the packet given by N in the current stream"

        #
        # Global set command, for debugger state.
        #

    def do_set(self, args):
        if len(args) <= 0:
            self.help_set()
            return
        try:
            (key, value) = args.split()
        except:
            self.help_set()
            return

        if key == "list_length":
            self.options.list_length = int(value)
        elif key == "layer":
            value = int(value)
            if value < 0:
                self.options.layer = None
            if value > 7:
                self.help_set()
            else:
                self.options.layer = value
        elif key == "current":
            if (value == None):
                print "Must supply a number between 0 and %d" % len(self.streams - 1)
                self.help_set()
                return
            value = int(value)
            if value < 0:
                print "Must be greater than 0"
                self.help_set()
                return
            if value >= len(self.streams):
                print "Must be less than %d" % (len(self.streams) - 1)
                self.help_set()
                return
            else:
                self.current = self.streams[value]

    def complete_set(self, text, line, begidx, endidx):
        """Completion for the set command"""
        if (text in ["li", "lis", "list", "list_", "list_l", "list_le", "list_len", "list_leng", "list_lengt"]):
            return ["list_length"]
        elif (text in ["la", "lay", "laye"]):
            return ["layer"]
        elif (text in ["c", "cu", "cur", "curr", "curre", "curren"]):
            return ["current"]
        else:
            if (line == "set current "):
                stream_list = []
                for i in range(0, len(self.streams)):
                    stream_list.append(str(i))
                return stream_list

    def help_set(self):
        print "set option value\n\noptions: list_length - how many packets to list at one time.\n         layer - ISO layer to show, -1 shows all\n         current - set the current stream we're inspecting [0..n]"

    def do_show(self, args):
        """Show the current options that are set."""
        print self.options

        #
        # Stream manipulation commands.  Breakpoints, position changes
        # and packet manipluation.
        #

    def do_break(self, args):
        """set a breakpoint at the current index in the stream or at the numeric index given"""
        if (args == ""):
            numarg = self.current.position
        else:
            numarg = self.numarg(args)
        if (numarg == None):
            self.help_break()
            return
        self.current.add_break(numarg)

    def help_break(self):
        print "break (N)"
        print "set a breakpoint at the current index in the stream or at the numeric index given"

    def do_next(self, args):
        """Move to the next packet"""
        if (self.current == None):
            print "No current stream.  Use the create or set commands"
            return
        if (len(args) > 0):
            jump = self.numarg(args)
            if (jump == None):
                self.help_next()
                return
            else:
                self.current.next(jump)
        else:
            self.current.next()

    def help_next(self):
        print "next (N)"
        print "Move to the next packet in the list.  With a numeric argument, N, move N packets ahead in the list."

    def do_prev(self, args):
        """Move to the previous packet"""
        if (self.current == None):
            print "No current stream.  Use the create or set commands"
            return
        if (len(args) > 0):
            jump = self.numarg(args)
            if (jump == None):
                self.help_prev()
                return
            else:
                self.current.prev(jump)
        else:
            self.current.prev()

    def help_prev(self):
        print "prev (N)"
        print "Move to the previous packet in the list.  With a numeric argument, N, move N packets back in the list."

        #
        # Miscellaneous helper functions for our cli
        #

    def numarg(self, args):
        """If the argument passed is numeric pass back the number, otherwise pass back None"""
        if (type(args) != str):
            print "N must be a number"
            return None
        if (args.isdigit() != True):
            print "N must be a number"
            return None
        return int(args)

<|endoftext|>


<commit_before>import logging
import sys
import time
import json
from multiprocessing import Process

import pika

from scrapy.utils.project import get_project_settings

from task import crawl, gen_lxmlspider, gen_blogspider

settings = get_project_settings()

def run(ch, method, properties, body):
    args = json.loads(body)['spiders']
    p = Process(target=crawl,
                args=(args,))
    p.start()
    p.join()

def lxmlspider(ch, method, properties, body):
    args = json.loads(body)
    gen_lxmlspider(args)

def blogspider(ch, method, properties, body):
    args = json.loads(body)
    gen_blogspider(args)

def task(callback, queue):
    host = settings['BROKER_URL']
    connection = pika.BlockingConnection(pika.connection.URLParameters(host))
    channel = connection.channel()
    channel.queue_declare(queue=queue)
    channel.basic_consume(callback,
                          queue=queue,
                          no_ack=True)
    channel.start_consuming()

def init_logger(settings):
    root = logging.getLogger()
    root.setLevel(logging.DEBUG)
    handler = logging.StreamHandler(sys.stdout)
    handler.setLevel(settings['LOG_LEVEL'])
    handler.setFormatter(logging.Formatter(settings['LOG_FORMAT']))
    root.addHandler(handler)

def main():
    init_logger(settings)
    logger = logging.getLogger(__name__)
    TASKS = [(run, settings['CRAWL_QUEUE_NAME']),
             (lxmlspider, settings['LXMLSPIDER_QUEUE_NAME']),
             (blogspider, settings['BLOGSPIDER_QUEUE_NAME'])]
    consumers = [(Process(target=task, args=_), _) for _ in TASKS]
    time.sleep(60)
    for p, _ in consumers:
        p.start()

    logger.info('rpc task running ...')
    for i, (p, args) in enumerate(consumers):
        logger.info('check task state ...')
        if not p.is_alive():
            consumers[i] = Process(target=task,
                                   args=args)
        time.sleep(120)

if __name__ == '__main__':
    main()
<commit_msg>update logger formatter<commit_after>import logging
import sys
import time
import json
from multiprocessing import Process

import pika

from scrapy.utils.project import get_project_settings

from task import crawl, gen_lxmlspider, gen_blogspider

settings = get_project_settings()

def run(ch, method, properties, body):
    args = json.loads(body)['spiders']
    p = Process(target=crawl,
                args=(args,))
    p.start()
    p.join()

def lxmlspider(ch, method, properties, body):
    args = json.loads(body)
    gen_lxmlspider(args)

def blogspider(ch, method, properties, body):
    args = json.loads(body)
    gen_blogspider(args)

def task(callback, queue):
    host = settings['BROKER_URL']
    connection = pika.BlockingConnection(pika.connection.URLParameters(host))
    channel = connection.channel()
    channel.queue_declare(queue=queue)
    channel.basic_consume(callback,
                          queue=queue,
                          no_ack=True)
    channel.start_consuming()

def init_logger(settings):
    root = logging.getLogger()
    root.setLevel(logging.DEBUG)
    handler = logging.StreamHandler(sys.stdout)
    handler.setLevel(settings['LOG_LEVEL'])
    handler.setFormatter(logging.Formatter(settings['LOG_FORMAT'],
                                           settings['LOG_DATEFORMAT']))
    root.addHandler(handler)

def main():
    init_logger(settings)
    logger = logging.getLogger(__name__)
    TASKS = [(run, settings['CRAWL_QUEUE_NAME']),
             (lxmlspider, settings['LXMLSPIDER_QUEUE_NAME']),
             (blogspider, settings['BLOGSPIDER_QUEUE_NAME'])]
    consumers = [(Process(target=task, args=_), _) for _ in TASKS]
    time.sleep(60)
    for p, _ in consumers:
        p.start()

    logger.info('rpc task running ...')
    for i, (p, args) in enumerate(consumers):
        logger.info('check task state ...')
        if not p.is_alive():
            consumers[i] = Process(target=task,
                                   args=args)
        time.sleep(120)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
platformer.py
Author: Dina
Credit: so far none
Assignment:
Write and submit a program that implements the sandbox platformer game:
https://github.com/HHS-IntroProgramming/Platformer
"""
from ggame import App, Color, LineStyle, Sprite, RectangleAsset, CircleAsset, EllipseAsset, PolygonAsset

red= Color(0xFE2E64, 1.0)
bluelight = Color(0x81F7F3,1.0)
#blue= Color(0x0000ff, 1.0)
thinline = LineStyle(3, red)

class Wall(Sprite):
    wall = RectangleAsset(50, 50, thinline, bluelight)
    def __init__(self, xPos, yPos):
        super().__init__(Wall.wall, (xPos, yPos))
        self.x = xPos
        self.y = yPos
'''
class Ball(Sprite):
    ball = CircleAsset(15, thinline, blue)

def moveChara(event):
 Sprite(Ball, (event.x, event.y))
'''
def buildWall(event):
    x = event.x%30
    if x != 0:
        if x >= 15:
            e = 30- x
            x = event.x + e
        elif x < 15:
            e = 15-x
            x = event.x - e
    Wall(x-25, event.y-25)

class Platformer(App):
    def __init__(self):
        super().__init__()

myapp = Platformer()
myapp.listenMouseEvent('click', buildWall)
#myapp.listenKeyEvent('keydown', 'space', moveChara)
myapp.run()<commit_msg>Updated from Brython Server: 11/18/2015 1:55:21 PM<commit_after>"""
platformer.py
Author: Dina
Credit: so far none
Assignment:
Write and submit a program that implements the sandbox platformer game:
https://github.com/HHS-IntroProgramming/Platformer
"""
from ggame import App, Color, LineStyle, Sprite, RectangleAsset, CircleAsset, EllipseAsset, PolygonAsset

red= Color(0xFE2E64, 1.0)
bluelight = Color(0x81F7F3,1.0)
#blue= Color(0x0000ff, 1.0)
thinline = LineStyle(3, red)

class Wall(Sprite):
    wall = RectangleAsset(50, 50, thinline, bluelight)
    def __init__(self, xPos, yPos):
        super().__init__(Wall.wall, (xPos, yPos))
        self.x = xPos
        self.y = yPos

def buildWall(event):
    x = event.x- event.x%50
    y = event.y- event.y%50
    Wall(x-25, y-25)

class Platformer(App):
    def __init__(self):
        super().__init__()
'''
class Ball(Sprite):
    ball = CircleAsset(15, thinline, blue)

def moveChara(event):
 Sprite(Ball, (event.x, event.y))
'''
myapp = Platformer()
myapp.listenMouseEvent('click', buildWall)
#myapp.listenKeyEvent('keydown', 'space', moveChara)
myapp.run()<|endoftext|>


<commit_before>"""
platformer.py
Author: Emma Supattapone
Credit: Abby Feyrer, Mr. Dennison
Assignment:
Write and submit a program that implements the sandbox platformer game:
https://github.com/HHS-IntroProgramming/Platformer
"""

from ggame import App
myapp = App()

SCREEN_WIDTH = 600
SCREEN_HEIGHT = 400
from ggame import App, Color, LineStyle, Sprite, Frame
from ggame import RectangleAsset, CircleAsset, EllipseAsset, PolygonAsset

red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)
white = Color(0xffffff, 1.0)
orange = Color(0xffa500,1.0)
purple = Color(0xa020f0,1.0)
thinline = LineStyle(1, black)
notthinline = LineStyle(3, green)
mouseposition = (0, 0)
yum=0

#Wall
class block(Sprite):
    asset=RectangleAsset(50, 50, thinline, black)
    def __init__(self, position):
        super(). __init__(block.asset, position)

rectangle = RectangleAsset(50, 50, thinline, black)

rounding = lambda x: 5 * round(x/5, -1)
e = []

def w(event):
    e.append(block((rounding(mouseposition[0] - 25), rounding(mouseposition[1] - 25))))
    rectangle.y = mouseposition[1]
#    print(e)

def move(event):
    global mouseposition
#    print(event.x, event.y)
    mouseposition = (event.x, event.y)

#Spring
sprig = []
class blockoo(Sprite):
    asset=RectangleAsset(15,4, thinline, red)
    def __init__(self, position):
        super(). __init__(blockoo.asset, position)
        self.vx=1
        self.vy=1

def s(event):
    sprig.append(blockoo(((mouseposition[0]), (mouseposition[1]))))
    blockoo.x = mouseposition[0]
    blockoo.y = mouseposition[1]
#    print(blockoo.y)
#    print(sprig)

def collidingWithSpritess(blockoo, block):
    if blockoo is block:
        return False
    elif blockoo._collisionStyle == block._collisionStyle == type(blockoo)._circCollision:
        dist3 = (blockoo.x - block.x)**2 + (blockoo.y - block.y)**2
        return dist3 < (45)**2
    else:
        return (not (blockoo.xmin > block.xmax
            or blockoo.xmax < block.xmin
            or blockoo.ymin > block.ymax
            or blockoo.ymax < block.ymin))

blockoo.go = True

def move(event):
    global mouseposition
    mouseposition = (event.x, event.y)

#Player
class blocko(Sprite):
    asset=RectangleAsset(15,30, notthinline, purple)
    def __init__(self, position):
        super(). __init__(blocko.asset, position)
        self.vx = 1
        self.vy = 1

def p(event):
    player.x=(mouseposition[0])
    player.y=(mouseposition[1])
#    print(mouseposition[0], mouseposition[1])
    t=0
player = blocko((4000,1))
player.go = True
rectangelot = RectangleAsset(15,30, notthinline, purple)

def move(event):
    global mouseposition
#    print(event.x, event.y)
    mouseposition = (event.x, event.y)

def collidingWithSprites(blocko, block):
    if blocko is block:
        return False
    elif blocko._collisionStyle == block._collisionStyle == type(blocko)._circCollision:
        dist2 = (blocko.x - block.x)**2 + (blocko.y - block.y)**2
        return dist2 < (45)**2
    else:
        return (blocko.xmin > block.xmax
            or blocko.xmax < block.xmin
            or blocko.ymin > block.ymax
            or blocko.ymax < block.ymin)

myapp.run()

#class KeyEvent(_Event):
global star, yum
star = 0
def mup(event):
    star = 2
    tt=16
def might(event):
    star = 3
def meft(event):
    star = 4

global t
global tt
t=0
def step():
    global t, tt
    if player.go:
        cc = player.collidingWithSprites(block)
        if len(cc) > 0:
            print("pancakes")
            t=0
        else:
            t=t+0.05
            player.y = player.y + 3 * t
    if star == 2:
        if tt > -10:
            print("with syrup")
            player.y = player.y - 1.2**tt
            tt=tt-1
    elif star == 3:
        print("and blueberries")
        player.x = player.x + 10
        star=0
    elif star == 4:
        print("chocolate chip")
        player.x = player.x - 10
        star=0

"""
global e
def steep():
    global e
    if blockoo.go:
        print("mooooo")
        cc = player.collidingWithSprites(block)
        if len(cc) > 0:
            print("moo")
        else:
            print("cow")
            e=e+0.05
            blockoo.y = blockoo.y + 3 * e
myapp.run(steep)
"""
global g
g=0
def steppp():
    if blockoo.go:
        print("yes")
        global g
    for x in sprig:
        print(x)
        global g
        ccc = sprig.collidingWithSpritess(block)
        if len(ccc) > 0:
            print("aaaaa")
            g=0
        else:
            print("bbbb")
            g=g+0.05
            x.y = x.y + 3 * g

ee = [1,2,3,4,5,6,7,8]
for u in ee:
    print(u)

"""
    if star == 2:
        if tt > -10:
            print("with syrup")
            e.y = e.y - 1.2**tt
            tt=tt-1

    if blockoo.go:
        bb = blockoo.collidingWithSprites(block)
        if len(bb) > 0:
            print("blueberry muffins")
            g=0
        else:
            g=g+0.5
            blockoo.y = blockoo.y + 3 * g
"""
myapp.run(steppp)

myapp = App(SCREEN_WIDTH, SCREEN_HEIGHT)

myapp.run(step)

myapp.listenKeyEvent('keydown' , 'w' , w)
myapp.listenMouseEvent('mousemove', move)
myapp.listenKeyEvent('keydown' , 'p' , p)
myapp.listenKeyEvent('keydown' , 's' , s)
myapp.listenKeyEvent('keydown' , 'up arrow' , mup)
myapp.listenKeyEvent('keydown' , 'right arrow' , might)
myapp.listenKeyEvent('keydown' , 'left arrow' , meft)

"""
problems:
player jumping does not work
must stop player from sinking into the walls when falling
add gravity+spring ability to the springs
make the sides+bottom of the walls collide with the player rather than just the tops

is it different if it is a list?

"""

"""
smol = RectangleAsset(0.1,0.1,thinline, purple)
#E
Sprite(smol, (100,100))
Sprite(smol, (100,101))
Sprite(smol, (100,102))
Sprite(smol, (100,103))
Sprite(smol, (100,104))
Sprite(smol, (101,100))
Sprite(smol, (101,102))
Sprite(smol, (101,104))
Sprite(smol, (102,100))
Sprite(smol, (102,104))
#m1
Sprite(smol, (104,104))
Sprite(smol, (106,104))
Sprite(smol, (108,104))
Sprite(smol, (104,103))
Sprite(smol, (106,103))
Sprite(smol, (108,103))
Sprite(smol, (105,102))
Sprite(smol, (107,102))
#m2
Sprite(smol, (110,104))
Sprite(smol, (112,104))
Sprite(smol, (114,104))
Sprite(smol, (110,103))
Sprite(smol, (112,103))
Sprite(smol, (114,103))
Sprite(smol, (111,102))
Sprite(smol, (113,102))
#a
Sprite(smol, (117,102))
Sprite(smol, (118,102))
Sprite(smol, (116,103))
Sprite(smol, (117,104))
Sprite(smol, (118,103))
Sprite(smol, (119,104))
"""
<commit_msg>Updated from Brython Server: 6/7/2017 1:46:09 PM<commit_after>"""
platformer.py
Author: Emma Supattapone
Credit: Abby Feyrer, Mr. Dennison
Assignment:
Write and submit a program that implements the sandbox platformer game:
https://github.com/HHS-IntroProgramming/Platformer
"""

from ggame import App
myapp = App()

SCREEN_WIDTH = 600
SCREEN_HEIGHT = 400
from ggame import App, Color, LineStyle, Sprite, Frame
from ggame import RectangleAsset, CircleAsset, EllipseAsset, PolygonAsset

red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)
white = Color(0xffffff, 1.0)
orange = Color(0xffa500,1.0)
purple = Color(0xa020f0,1.0)
thinline = LineStyle(1, black)
notthinline = LineStyle(3, green)
mouseposition = (0, 0)
yum=0

#Wall
class block(Sprite):
    asset=RectangleAsset(50, 50, thinline, black)
    def __init__(self, position):
        super(). __init__(block.asset, position)

rectangle = RectangleAsset(50, 50, thinline, black)

rounding = lambda x: 5 * round(x/5, -1)
e = []

def w(event):
    e.append(block((rounding(mouseposition[0] - 25), rounding(mouseposition[1] - 25))))
    rectangle.y = mouseposition[1]
#    print(e)

def move(event):
    global mouseposition
#    print(event.x, event.y)
    mouseposition = (event.x, event.y)

#Spring
sprig = []
class blockoo(Sprite):
    asset=RectangleAsset(15,4, thinline, red)
    def __init__(self, position):
        super(). __init__(blockoo.asset, position)
        self.vx=1
        self.vy=1

def s(event):
    sprig.append(blockoo(((mouseposition[0]), (mouseposition[1]))))
    blockoo.x = mouseposition[0]
    blockoo.y = mouseposition[1]
#    print(blockoo.y)
#    print(sprig)

def collidingWithSpritess(blockoo, block):
    if blockoo is block:
        return False
    elif blockoo._collisionStyle == block._collisionStyle == type(blockoo)._circCollision:
        dist3 = (blockoo.x - block.x)**2 + (blockoo.y - block.y)**2
        return dist3 < (45)**2
    else:
        return (not (blockoo.xmin > block.xmax
            or blockoo.xmax < block.xmin
            or blockoo.ymin > block.ymax
            or blockoo.ymax < block.ymin))

blockoo.go = True

def move(event):
    global mouseposition
    mouseposition = (event.x, event.y)

#Player
class blocko(Sprite):
    asset=RectangleAsset(15,30, notthinline, purple)
    def __init__(self, position):
        super(). __init__(blocko.asset, position)
        self.vx = 1
        self.vy = 1

def p(event):
    player.x=(mouseposition[0])
    player.y=(mouseposition[1])
#    print(mouseposition[0], mouseposition[1])
    t=0
player = blocko((4000,1))
player.go = True
rectangelot = RectangleAsset(15,30, notthinline, purple)

def move(event):
    global mouseposition
#    print(event.x, event.y)
    mouseposition = (event.x, event.y)

def collidingWithSprites(blocko, block):
    if blocko is block:
        return False
    elif blocko._collisionStyle == block._collisionStyle == type(blocko)._circCollision:
        dist2 = (blocko.x - block.x)**2 + (blocko.y - block.y)**2
        return dist2 < (45)**2
    else:
        return (blocko.xmin > block.xmax
            or blocko.xmax < block.xmin
            or blocko.ymin > block.ymax
            or blocko.ymax < block.ymin)

myapp.run()

#class KeyEvent(_Event):
global star, yum
star = 0
def mup(event):
    star = 2
    tt=16
def might(event):
    star = 3
def meft(event):
    star = 4

global t
global tt
t=0
def step():
    global t, tt
    if player.go:
        cc = player.collidingWithSprites(block)
        if len(cc) > 0:
            print("pancakes")
            t=0
        else:
            t=t+0.05
            player.y = player.y + 3 * t
    if star == 2:
        if tt > -10:
            print("with syrup")
            player.y = player.y - 1.4**tt
            tt=tt-1
    elif star == 3:
        print("and blueberries")
        player.x = player.x + 10
        star=0
    elif star == 4:
        print("chocolate chip")
        player.x = player.x - 10
        star=0

"""
global e
def steep():
    global e
    if blockoo.go:
        print("mooooo")
        cc = player.collidingWithSprites(block)
        if len(cc) > 0:
            print("moo")
        else:
            print("cow")
            e=e+0.05
            blockoo.y = blockoo.y + 3 * e
myapp.run(steep)
"""
global g
g=0
def steppp():
    if blockoo.go:
        print("yes")
        global g
    for x in sprig:
        print(x)
        print("mmmm")
        global g
        ccc = x.collidingWithSprites(block)
        print("tyjr")
        if len(ccc) > 0:
            print("aaaaa")
            g=0
        else:
            print("bbbb")
            g=g+0.05
            x.y = x.y + 3 * g

"""
    if star == 2:
        if tt > -10:
            print("with syrup")
            e.y = e.y - 1.2**tt
            tt=tt-1

    if blockoo.go:
        bb = blockoo.collidingWithSprites(block)
        if len(bb) > 0:
            print("blueberry muffins")
            g=0
        else:
            g=g+0.5
            blockoo.y = blockoo.y + 3 * g
"""
myapp.run(steppp)

myapp = App(SCREEN_WIDTH, SCREEN_HEIGHT)

myapp.run(step)

myapp.listenKeyEvent('keydown' , 'w' , w)
myapp.listenMouseEvent('mousemove', move)
myapp.listenKeyEvent('keydown' , 'p' , p)
myapp.listenKeyEvent('keydown' , 's' , s)
myapp.listenKeyEvent('keydown' , 'up arrow' , mup)
myapp.listenKeyEvent('keydown' , 'right arrow' , might)
myapp.listenKeyEvent('keydown' , 'left arrow' , meft)

"""
problems:
player jumping does not work
must stop player from sinking into the walls when falling
add gravity+spring ability to the springs
make the sides+bottom of the walls collide with the player rather than just the tops

is it different if it is a list?

"""

"""
smol = RectangleAsset(0.1,0.1,thinline, purple)
#E
Sprite(smol, (100,100))
Sprite(smol, (100,101))
Sprite(smol, (100,102))
Sprite(smol, (100,103))
Sprite(smol, (100,104))
Sprite(smol, (101,100))
Sprite(smol, (101,102))
Sprite(smol, (101,104))
Sprite(smol, (102,100))
Sprite(smol, (102,104))
#m1
Sprite(smol, (104,104))
Sprite(smol, (106,104))
Sprite(smol, (108,104))
Sprite(smol, (104,103))
Sprite(smol, (106,103))
Sprite(smol, (108,103))
Sprite(smol, (105,102))
Sprite(smol, (107,102))
#m2
Sprite(smol, (110,104))
Sprite(smol, (112,104))
Sprite(smol, (114,104))
Sprite(smol, (110,103))
Sprite(smol, (112,103))
Sprite(smol, (114,103))
Sprite(smol, (111,102))
Sprite(smol, (113,102))
#a
Sprite(smol, (117,102))
Sprite(smol, (118,102))
Sprite(smol, (116,103))
Sprite(smol, (117,104))
Sprite(smol, (118,103))
Sprite(smol, (119,104))
"""
<|endoftext|>


<commit_before>from __future__ import unicode_literals
__author__ = 'luissaguas'
from frappe import _
import frappe
import json
from urllib2 import unquote
import logging, time
from frappe.utils import cint

from frappe.core.doctype.communication.communication import make

import JasperRoot as Jr
from jasper_erpnext_report import jasper_session_obj
from jasper_erpnext_report.utils.jasper_email import sendmail
from jasper_erpnext_report.core.JasperRoot import get_copies

_logger = logging.getLogger(frappe.__name__)

def boot_session(bootinfo):
	#bootinfo['jasper_server_info'] = get_server_info()
	bootinfo['jasper_reports_list'] = get_reports_list_for_all()

@frappe.whitelist()
def get_reports_list_for_all():
	jsr = jasper_session_obj or Jr.JasperRoot()
	return jsr.get_reports_list_for_all()

@frappe.whitelist()
def get_reports_list(doctype, docnames):
	jsr = jasper_session_obj or Jr.JasperRoot()
	return jsr.get_reports_list(doctype, docnames)

@frappe.whitelist()
def report_polling(data):
	jsr = jasper_session_obj or Jr.JasperRoot()
	return jsr.report_polling(data)

@frappe.whitelist()
def get_report(data):
	#print "data get_reportssss {}".format(unquote(data))
	if not data:
		frappe.throw(_("No data for this Report!!!"))
	if isinstance(data, basestring):
		data = json.loads(unquote(data))
	pformat = data.get("pformat")
	fileName, content = _get_report(data)
	make_pdf(fileName, content, pformat)

#def _get_report(data, merge_all=True, pages=None, email=False):
def _get_report(data):
	jsr = jasper_session_obj or Jr.JasperRoot()
	fileName, content = jsr.get_report_server(data)
	#file_name, output = jsr.make_pdf(fileName, content, pformat, merge_all, pages)
	#if not email:
	#	jsr.prepare_file_to_client(file_name, output)
	#	return

	#return file_name, output
	return fileName, content

def make_pdf(fileName, content, pformat, merge_all=True, pages=None, email=False):
	jsr = jasper_session_obj or Jr.JasperRoot()
	file_name, output = jsr.make_pdf(fileName, content, pformat, merge_all, pages)
	if not email:
		jsr.prepare_file_to_client(file_name, output)
		return

	return file_name, output

@frappe.whitelist()
def run_report(data, docdata=None, rtype="Form"):
	from frappe.utils import pprint_dict
	if not data:
		frappe.throw("No data for this Report!!!")
	if isinstance(data, basestring):
		data = json.loads(data)
	jsr = jasper_session_obj or Jr.JasperRoot()
	print "params in run_report 2 {}".format(pprint_dict(data))
	return jsr.run_report(data, docdata=docdata, rtype=rtype)

@frappe.whitelist()
def get_server_info():
	jsr = jasper_session_obj or Jr.JasperRoot()
	return jsr._get_server_info()

@frappe.whitelist()
def jasper_server_login():
	jsr = jasper_session_obj or Jr.JasperRoot()
	return jsr.login()

@frappe.whitelist()
def get_doc(doctype, docname):
	import jasper_erpnext_report.utils.utils as utils
	data = {}
	doc = frappe.get_doc(doctype, docname)
	if utils.check_jasper_perm(doc.get("jasper_roles", None)):
		data = {"data": doc}
	frappe.local.response.update(data)

@frappe.whitelist()
def jasper_make(doctype=None, name=None, content=None, subject=None, sent_or_received = "Sent",
	sender=None, recipients=None, communication_medium="Email", send_email=False,
	print_html=None, print_format=None, attachments='[]', send_me_a_copy=False, set_lead=True, date=None,
	jasper_doc=None, docdata=None, rtype="Form"):

	jasper_polling_time = frappe.db.get_value('JasperServerConfig', fieldname="jasper_polling_time")
	data = json.loads(jasper_doc)
	result = run_report(data, docdata, rtype)
	if result[0].get("status", "not ready") != "ready":
		poll_data = prepare_polling(result)
		result = report_polling(poll_data)
		limit = 0
		#while result[0].get("status", "not ready") != "ready" and limit <= 10:
		while not "status" in result[0] and limit <= 10:
			time.sleep(cint(jasper_polling_time)/1000)
			result = report_polling(poll_data)
			limit += 1

	#we have to remove the original and send only duplicate
	if result[0].get("status", "not ready") == "ready":
		pformat = data.get("pformat")
		rdoc = frappe.get_doc(data.get("doctype"), data.get('report_name'))
		ncopies = get_copies(rdoc, pformat)
		#file_name, output = _get_report(result[0], merge_all=True, pages=None, email=True)
		fileName, jasper_content = _get_report(result[0])
		merge_all = True
		pages = None
		if pformat == "pdf" and ncopies > 1:
			merge_all = False
			pages = get_pages(ncopies, len(jasper_content))

		file_name, output = make_pdf(fileName, jasper_content, pformat, merge_all=merge_all, pages=pages, email=True)
	else:
		print "not sent by email... {}".format(result)
		frappe.throw(_("Error generating PDF, try again later"))
		frappe.errprint(frappe.get_traceback())
		return

	#attach = jasper_make_attach(data, file_name, output, attachments, result)

	make(doctype=doctype, name=name, content=content, subject=subject, sent_or_received=sent_or_received,
		sender=sender, recipients=recipients, communication_medium=communication_medium, send_email=False,
		print_html=print_html, print_format=print_format, attachments=attachments, send_me_a_copy=send_me_a_copy, set_lead=set_lead,
		date=date)

	sendmail(data, file_name, output, result[0].get("requestId"), doctype=doctype, name=name, content=content, subject=subject, sent_or_received=sent_or_received,
		sender=sender, recipients=recipients, print_html=print_html, print_format=print_format, attachments=attachments,
		send_me_a_copy=send_me_a_copy)

"""
def jasper_make_attach(data, file_name, output, attachments, result):

	from frappe.utils.file_manager import get_site_path
	path_join = os.path.join
	#rdoc = frappe.get_doc(data.get("doctype"), data.get('report_name'))
	#for_all_sites = rdoc.jasper_all_sites_report
	#jasper_path = get_jasper_path(for_all_sites)
	public = get_site_path("public")
	jasper_path_intern = path_join("jasper_sent_email", result[0].get("requestId"))
	outputPath = path_join(public, jasper_path_intern)
	frappe.create_folder(outputPath)
	file_path = path_join(outputPath, file_name)
	file_path_intern = path_join(jasper_path_intern, file_name)
	write_StringIO_to_file(file_path, output)

	attach = json.loads(attachments)
	attach.append(file_path_intern)
	print "sent by email 2... {} attach {}".format(file_path_intern, attach)

	return json.dumps(attach)
"""

def prepare_polling(data):
	reqids = []
	for d in data:
		reqids.append(d.get("requestId"))
	poll_data = {"reqIds": reqids, "reqtime": d.get("reqtime"), "pformat": d.get("pformat"), "origin": d.get("origin")}
	return poll_data

def get_pages(ncopies, total_pages):
	pages = []
	clientes = total_pages/ncopies
	for n in range(clientes - 1):
		pages.append(n*ncopies)

	return pages

<commit_msg>send docs by email<commit_after>from __future__ import unicode_literals
__author__ = 'luissaguas'
from frappe import _
import frappe
import json
from urllib2 import unquote
import logging, time
from frappe.utils import cint

from frappe.core.doctype.communication.communication import make

import JasperRoot as Jr
from jasper_erpnext_report import jasper_session_obj
from jasper_erpnext_report.utils.jasper_email import sendmail
from jasper_erpnext_report.core.JasperRoot import get_copies

_logger = logging.getLogger(frappe.__name__)

def boot_session(bootinfo):
	#bootinfo['jasper_server_info'] = get_server_info()
	bootinfo['jasper_reports_list'] = get_reports_list_for_all()

@frappe.whitelist()
def get_reports_list_for_all():
	jsr = jasper_session_obj or Jr.JasperRoot()
	return jsr.get_reports_list_for_all()

@frappe.whitelist()
def get_reports_list(doctype, docnames):
	jsr = jasper_session_obj or Jr.JasperRoot()
	return jsr.get_reports_list(doctype, docnames)

@frappe.whitelist()
def report_polling(data):
	jsr = jasper_session_obj or Jr.JasperRoot()
	return jsr.report_polling(data)

@frappe.whitelist()
def get_report(data):
	#print "data get_reportssss {}".format(unquote(data))
	if not data:
		frappe.throw(_("No data for this Report!!!"))
	if isinstance(data, basestring):
		data = json.loads(unquote(data))
	pformat = data.get("pformat")
	fileName, content = _get_report(data)
	make_pdf(fileName, content, pformat)

#def _get_report(data, merge_all=True, pages=None, email=False):
def _get_report(data):
	jsr = jasper_session_obj or Jr.JasperRoot()
	fileName, content = jsr.get_report_server(data)
	#file_name, output = jsr.make_pdf(fileName, content, pformat, merge_all, pages)
	#if not email:
	#	jsr.prepare_file_to_client(file_name, output)
	#	return

	#return file_name, output
	return fileName, content

def make_pdf(fileName, content, pformat, merge_all=True, pages=None, email=False):
	jsr = jasper_session_obj or Jr.JasperRoot()
	file_name, output = jsr.make_pdf(fileName, content, pformat, merge_all, pages)
	if not email:
		jsr.prepare_file_to_client(file_name, output)
		return

	return file_name, output

@frappe.whitelist()
def run_report(data, docdata=None, rtype="Form"):
	from frappe.utils import pprint_dict
	if not data:
		frappe.throw("No data for this Report!!!")
	if isinstance(data, basestring):
		data = json.loads(data)
	jsr = jasper_session_obj or Jr.JasperRoot()
	print "params in run_report 2 {}".format(pprint_dict(data))
	return jsr.run_report(data, docdata=docdata, rtype=rtype)

@frappe.whitelist()
def get_server_info():
	jsr = jasper_session_obj or Jr.JasperRoot()
	return jsr._get_server_info()

@frappe.whitelist()
def jasper_server_login():
	jsr = jasper_session_obj or Jr.JasperRoot()
	return jsr.login()

@frappe.whitelist()
def get_doc(doctype, docname):
	import jasper_erpnext_report.utils.utils as utils
	data = {}
	doc = frappe.get_doc(doctype, docname)
	if utils.check_jasper_perm(doc.get("jasper_roles", None)):
		data = {"data": doc}
	frappe.local.response.update(data)

@frappe.whitelist()
def jasper_make(doctype=None, name=None, content=None, subject=None, sent_or_received = "Sent",
	sender=None, recipients=None, communication_medium="Email", send_email=False,
	print_html=None, print_format=None, attachments='[]', send_me_a_copy=False, set_lead=True, date=None,
	jasper_doc=None, docdata=None, rtype="Form"):

	jasper_polling_time = frappe.db.get_value('JasperServerConfig', fieldname="jasper_polling_time")
	data = json.loads(jasper_doc)
	result = run_report(data, docdata, rtype)
	if result[0].get("status", "not ready") != "ready":
		poll_data = prepare_polling(result)
		result = report_polling(poll_data)
		limit = 0
		#while result[0].get("status", "not ready") != "ready" and limit <= 10:
		while not "status" in result[0] and limit <= 10:
			time.sleep(cint(jasper_polling_time)/1000)
			result = report_polling(poll_data)
			limit += 1

	#we have to remove the original and send only duplicate
	if result[0].get("status", "not ready") == "ready":
		pformat = data.get("pformat")
		rdoc = frappe.get_doc(data.get("doctype"), data.get('report_name'))
		ncopies = get_copies(rdoc, pformat)
		#file_name, output = _get_report(result[0], merge_all=True, pages=None, email=True)
		fileName, jasper_content = _get_report(result[0])
		merge_all = True
		pages = None
		if pformat == "pdf" and ncopies > 1:
			merge_all = False
			pages = get_pages(ncopies, len(jasper_content))

		file_name, output = make_pdf(fileName, jasper_content, pformat, merge_all=merge_all, pages=pages, email=True)
	else:
		print "not sent by email... {}".format(result)
		frappe.throw(_("Error generating PDF, try again later"))
		frappe.errprint(frappe.get_traceback())
		return

	#attach = jasper_make_attach(data, file_name, output, attachments, result)

	make(doctype=doctype, name=name, content=content, subject=subject, sent_or_received=sent_or_received,
		sender=sender, recipients=recipients, communication_medium=communication_medium, send_email=False,
		print_html=print_html, print_format=print_format, attachments=attachments, send_me_a_copy=send_me_a_copy, set_lead=set_lead,
		date=date)

	sendmail(data, file_name, output, result[0].get("requestId"), doctype=doctype, name=name, content=content, subject=subject, sent_or_received=sent_or_received,
		sender=sender, recipients=recipients, print_html=print_html, print_format=print_format, attachments=attachments,
		send_me_a_copy=send_me_a_copy)

"""
def jasper_make_attach(data, file_name, output, attachments, result):

	from frappe.utils.file_manager import get_site_path
	path_join = os.path.join
	#rdoc = frappe.get_doc(data.get("doctype"), data.get('report_name'))
	#for_all_sites = rdoc.jasper_all_sites_report
	#jasper_path = get_jasper_path(for_all_sites)
	public = get_site_path("public")
	jasper_path_intern = path_join("jasper_sent_email", result[0].get("requestId"))
	outputPath = path_join(public, jasper_path_intern)
	frappe.create_folder(outputPath)
	file_path = path_join(outputPath, file_name)
	file_path_intern = path_join(jasper_path_intern, file_name)
	write_StringIO_to_file(file_path, output)

	attach = json.loads(attachments)
	attach.append(file_path_intern)
	print "sent by email 2... {} attach {}".format(file_path_intern, attach)

	return json.dumps(attach)
"""

def prepare_polling(data):
	reqids = []
	for d in data:
		reqids.append(d.get("requestId"))
	poll_data = {"reqIds": reqids, "reqtime": d.get("reqtime"), "pformat": d.get("pformat"), "origin": d.get("origin")}
	return poll_data

def get_pages(ncopies, total_pages):
	pages = []
	clientes = total_pages/ncopies
	for n in range(clientes):
		pages.append(n*ncopies)

	return pages

<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright (c) 2015 Orange
# guyrodrigue.koffi@orange.com
# morgan.richomme@orange.com
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
#
import re, json, os, urllib2, argparse, logging, yaml

with open('../functest.yaml') as f:
    functest_yaml = yaml.safe_load(f)
f.close()

HOME = os.environ['HOME']+"/"
SCENARIOS_DIR = HOME + functest_yaml.get("general").get("directories").get("dir_rally_scn")
RESULTS_DIR = HOME + functest_yaml.get("general").get("directories").get("dir_rally_res")

""" tests configuration """
tests = ['authenticate', 'glance', 'cinder', 'heat', 'keystone', 'neutron', 'nova', 'quotas', 'requests', 'tempest', 'vm', 'all', 'smoke']
parser = argparse.ArgumentParser()
parser.add_argument("test_name", help="The name of the test you want to perform with rally. "
                                      "Possible values are : "
                                      "[ {d[0]} | {d[1]} | {d[2]} | {d[3]} | {d[4]} | {d[5]} | {d[6]} "
                                      "| {d[7]} | {d[8]} | {d[9]} | {d[10]} | {d[11]} | {d[12]}]. The 'all' value performs all the tests scenarios "
                                      "except 'tempest'".format(d=tests))

parser.add_argument("-d", "--debug", help="Debug mode",  action="store_true")
parser.add_argument("test_mode", help="Tempest test mode", nargs='?', default="smoke")
args = parser.parse_args()
test_mode=args.test_mode

if not args.test_name == "tempest":
    if not args.test_mode == "smoke":
        parser.error("test_mode is only used with tempest")

""" logging configuration """
logger = logging.getLogger('run_rally')
logger.setLevel(logging.DEBUG)

ch = logging.StreamHandler()
if args.debug:
    ch.setLevel(logging.DEBUG)
else:
    ch.setLevel(logging.INFO)

formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)

def get_tempest_id(cmd_raw):
    """
    get task id from command rally result
    :param cmd_raw:
    :return: task_id as string
    """
    taskid_re = re.compile('^Verification UUID: (.*)$')
    for line in cmd_raw.splitlines(True):
        line = line.strip()
    match = taskid_re.match(line)

    if match:
        return match.group(1)
    return None

def get_task_id(cmd_raw):
    """
    get task id from command rally result
    :param cmd_raw:
    :return: task_id as string
    """
    taskid_re = re.compile('^Task +(.*): started$')
    for line in cmd_raw.splitlines(True):
        line = line.strip()
        match = taskid_re.match(line)
        if match:
            return match.group(1)
    return None

def task_succeed(json_raw):
    """
    Parse JSON from rally JSON results
    :param json_raw:
    :return: Bool
    """
    rally_report = json.loads(json_raw)
    rally_report = rally_report[0]
    if rally_report is None:
        return False
    if rally_report.get('result') is None:
        return False

    for result in rally_report.get('result'):
        if len(result.get('errors')) > 0:
            return False

    return True

def run_tempest():
    """
    the function dedicated to Tempest (functional tests for OpenStack)
    :param test_mode: Tempest mode smoke (default), full, ..
    :return: void
    """
    logger.info('starting {} Tempest ...'.format(test_mode))

    """ get the date """
    cmd = os.popen("date '+%d%m%Y_%H%M'")
    test_date = cmd.read().rstrip()

    cmd_line = "rally verify start {}".format(test_mode)
    logger.debug('running command line : {}'.format(cmd_line))
    cmd = os.popen(cmd_line)
    task_id = get_tempest_id(cmd.read())
    logger.debug('task_id : {}'.format(task_id))

    if task_id is None:
        logger.error("failed to retrieve task_id")
    exit(-1)

    """ check for result directory and create it otherwise """
    if not os.path.exists(RESULTS_DIR):
        logger.debug('does not exists, we create it'.format(RESULTS_DIR))
        os.makedirs(RESULTS_DIR)

    """ write log report file """
    report_file_name = '{}opnfv-tempest-{}.log'.format(RESULTS_DIR, test_date)
    cmd_line = "rally verify detailed {} > {} ".format(task_id, report_file_name)
    logger.debug('running command line : {}'.format(cmd_line))
    os.popen(cmd_line)

def run_task(test_name):
    """
    the "main" function of the script who lunch rally for a task
    :param test_name: name for the rally test
    :return: void
    """
    logger.info('starting {} test ...'.format(test_name))

    """ get the date """
    cmd = os.popen("date '+%d%m%Y_%H%M'")
    test_date = cmd.read().rstrip()

    """ check directory for scenarios test files or retrieve from git otherwise"""
    proceed_test = True
    test_file_name = '{}opnfv-{}.json'.format(SCENARIOS_DIR, test_name)
    if not os.path.exists(test_file_name):
        logger.debug('{} does not exists'.format(test_file_name))
        proceed_test = retrieve_test_cases_file(test_name, SCENARIOS_DIR)

    """ we do the test only if we have a scenario test file """
    if proceed_test:
        logger.debug('Scenario fetched from : {}'.format(test_file_name))
        cmd_line = "rally task start --abort-on-sla-failure %s" % test_file_name
        logger.debug('running command line : {}'.format(cmd_line))
        cmd = os.popen(cmd_line)
        task_id = get_task_id(cmd.read())
        logger.debug('task_id : {}'.format(task_id))

        if task_id is None:
            logger.error("failed to retrieve task_id")
            exit(-1)

        """ check for result directory and create it otherwise """
        if not os.path.exists(RESULTS_DIR):
            logger.debug('does not exists, we create it'.format(RESULTS_DIR))
            os.makedirs(RESULTS_DIR)

        """ write html report file """
        report_file_name = '{}opnfv-{}-{}.html'.format(RESULTS_DIR, test_name, test_date)
        cmd_line = "rally task report %s --out %s" % (task_id, report_file_name)
        logger.debug('running command line : {}'.format(cmd_line))
        os.popen(cmd_line)

        """ get and save rally operation JSON result """
        cmd_line = "rally task results %s" % task_id
        logger.debug('running command line : {}'.format(cmd_line))
        cmd = os.popen(cmd_line)
        json_results = cmd.read()
        with open('{}opnfv-{}-{}.json'.format(RESULTS_DIR, test_name, test_date), 'w') as f:
            logger.debug('saving json file')
            f.write(json_results)
            logger.debug('saving json file2')

        """ parse JSON operation result """
        if task_succeed(json_results):
            print 'Test OK'
        else:
            print 'Test KO'
    else:
        logger.error('{} test failed, unable to fetch a scenario test file'.format(test_name))

def retrieve_test_cases_file(test_name, tests_path):
    """
    Retrieve from github the sample test files
    :return: Boolean that indicates the retrieval status
    """

    """ do not add the "/" at the end """
    url_base = "https://git.opnfv.org/cgit/functest/plain/testcases/VIM/OpenStack/CI/suites"

    test_file_name = 'opnfv-{}.json'.format(test_name)
    logger.info('fetching {}/{} ...'.format(url_base, test_file_name))

    try:
        response = urllib2.urlopen('{}/{}'.format(url_base, test_file_name))
    except (urllib2.HTTPError, urllib2.URLError):
        return False
    file_raw = response.read()

    """ check if the test path exist otherwise we create it """
    if not os.path.exists(tests_path):
        os.makedirs(tests_path)

    with open('{}/{}'.format(tests_path, test_file_name), 'w') as f:
        f.write(file_raw)
    return True

def main():
    """ configure script """
    if not (args.test_name in tests):
        logger.error('argument not valid')
        exit(-1)

    if args.test_name == "all":
        for test_name in tests:
            if not (test_name == 'all' or test_name == 'tempest'):
                print(test_name)
                run_task(test_name)
    else:
        print(args.test_name)
        if args.test_name == 'tempest':
            run_tempest()
        else:
            run_task(args.test_name)

if __name__ == '__main__':
    main()
<commit_msg>fix bug on error detection<commit_after>#!/usr/bin/env python
#
# Copyright (c) 2015 Orange
# guyrodrigue.koffi@orange.com
# morgan.richomme@orange.com
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
#
import re, json, os, urllib2, argparse, logging, yaml

with open('../functest.yaml') as f:
    functest_yaml = yaml.safe_load(f)
f.close()

HOME = os.environ['HOME']+"/"
SCENARIOS_DIR = HOME + functest_yaml.get("general").get("directories").get("dir_rally_scn")
RESULTS_DIR = HOME + functest_yaml.get("general").get("directories").get("dir_rally_res")

""" tests configuration """
tests = ['authenticate', 'glance', 'cinder', 'heat', 'keystone', 'neutron', 'nova', 'quotas', 'requests', 'tempest', 'vm', 'all', 'smoke']
parser = argparse.ArgumentParser()
parser.add_argument("test_name", help="The name of the test you want to perform with rally. "
                                      "Possible values are : "
                                      "[ {d[0]} | {d[1]} | {d[2]} | {d[3]} | {d[4]} | {d[5]} | {d[6]} "
                                      "| {d[7]} | {d[8]} | {d[9]} | {d[10]} | {d[11]} | {d[12]}]. The 'all' value performs all the tests scenarios "
                                      "except 'tempest'".format(d=tests))

parser.add_argument("-d", "--debug", help="Debug mode",  action="store_true")
parser.add_argument("test_mode", help="Tempest test mode", nargs='?', default="smoke")
args = parser.parse_args()
test_mode=args.test_mode

if not args.test_name == "tempest":
    if not args.test_mode == "smoke":
        parser.error("test_mode is only used with tempest")

""" logging configuration """
logger = logging.getLogger('run_rally')
logger.setLevel(logging.DEBUG)

ch = logging.StreamHandler()
if args.debug:
    ch.setLevel(logging.DEBUG)
else:
    ch.setLevel(logging.INFO)

formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)

def get_tempest_id(cmd_raw):
    """
    get task id from command rally result
    :param cmd_raw:
    :return: task_id as string
    """
    taskid_re = re.compile('^Verification UUID: (.*)$')
    for line in cmd_raw.splitlines(True):
        line = line.strip()
    match = taskid_re.match(line)

    if match:
        return match.group(1)
    return None

def get_task_id(cmd_raw):
    """
    get task id from command rally result
    :param cmd_raw:
    :return: task_id as string
    """
    taskid_re = re.compile('^Task +(.*): started$')
    for line in cmd_raw.splitlines(True):
        line = line.strip()
        match = taskid_re.match(line)
        if match:
            return match.group(1)
    return None

def task_succeed(json_raw):
    """
    Parse JSON from rally JSON results
    :param json_raw:
    :return: Bool
    """
    rally_report = json.loads(json_raw)
    rally_report = rally_report[0]
    if rally_report is None:
        return False
    if rally_report.get('result') is None:
        return False

    for result in rally_report.get('result'):
        if len(result.get('error')) > 0:
            return False

    return True

def run_tempest():
    """
    the function dedicated to Tempest (functional tests for OpenStack)
    :param test_mode: Tempest mode smoke (default), full, ..
    :return: void
    """
    logger.info('starting {} Tempest ...'.format(test_mode))

    """ get the date """
    cmd = os.popen("date '+%d%m%Y_%H%M'")
    test_date = cmd.read().rstrip()

    cmd_line = "rally verify start {}".format(test_mode)
    logger.debug('running command line : {}'.format(cmd_line))
    cmd = os.popen(cmd_line)
    task_id = get_tempest_id(cmd.read())
    logger.debug('task_id : {}'.format(task_id))

    if task_id is None:
        logger.error("failed to retrieve task_id")
    exit(-1)

    """ check for result directory and create it otherwise """
    if not os.path.exists(RESULTS_DIR):
        logger.debug('does not exists, we create it'.format(RESULTS_DIR))
        os.makedirs(RESULTS_DIR)

    """ write log report file """
    report_file_name = '{}opnfv-tempest-{}.log'.format(RESULTS_DIR, test_date)
    cmd_line = "rally verify detailed {} > {} ".format(task_id, report_file_name)
    logger.debug('running command line : {}'.format(cmd_line))
    os.popen(cmd_line)

def run_task(test_name):
    """
    the "main" function of the script who lunch rally for a task
    :param test_name: name for the rally test
    :return: void
    """
    logger.info('starting {} test ...'.format(test_name))

    """ get the date """
    cmd = os.popen("date '+%d%m%Y_%H%M'")
    test_date = cmd.read().rstrip()

    """ check directory for scenarios test files or retrieve from git otherwise"""
    proceed_test = True
    test_file_name = '{}opnfv-{}.json'.format(SCENARIOS_DIR, test_name)
    if not os.path.exists(test_file_name):
        logger.debug('{} does not exists'.format(test_file_name))
        proceed_test = retrieve_test_cases_file(test_name, SCENARIOS_DIR)

    """ we do the test only if we have a scenario test file """
    if proceed_test:
        logger.debug('Scenario fetched from : {}'.format(test_file_name))
        cmd_line = "rally task start --abort-on-sla-failure %s" % test_file_name
        logger.debug('running command line : {}'.format(cmd_line))
        cmd = os.popen(cmd_line)
        task_id = get_task_id(cmd.read())
        logger.debug('task_id : {}'.format(task_id))

        if task_id is None:
            logger.error("failed to retrieve task_id")
            exit(-1)

        """ check for result directory and create it otherwise """
        if not os.path.exists(RESULTS_DIR):
            logger.debug('does not exists, we create it'.format(RESULTS_DIR))
            os.makedirs(RESULTS_DIR)

        """ write html report file """
        report_file_name = '{}opnfv-{}-{}.html'.format(RESULTS_DIR, test_name, test_date)
        cmd_line = "rally task report %s --out %s" % (task_id, report_file_name)
        logger.debug('running command line : {}'.format(cmd_line))
        os.popen(cmd_line)

        """ get and save rally operation JSON result """
        cmd_line = "rally task results %s" % task_id
        logger.debug('running command line : {}'.format(cmd_line))
        cmd = os.popen(cmd_line)
        json_results = cmd.read()
        with open('{}opnfv-{}-{}.json'.format(RESULTS_DIR, test_name, test_date), 'w') as f:
            logger.debug('saving json file')
            f.write(json_results)
            logger.debug('saving json file2')

        """ parse JSON operation result """
        if task_succeed(json_results):
            print 'Test OK'
        else:
            print 'Test KO'
    else:
        logger.error('{} test failed, unable to fetch a scenario test file'.format(test_name))

def retrieve_test_cases_file(test_name, tests_path):
    """
    Retrieve from github the sample test files
    :return: Boolean that indicates the retrieval status
    """

    """ do not add the "/" at the end """
    url_base = "https://git.opnfv.org/cgit/functest/plain/testcases/VIM/OpenStack/CI/suites"

    test_file_name = 'opnfv-{}.json'.format(test_name)
    logger.info('fetching {}/{} ...'.format(url_base, test_file_name))

    try:
        response = urllib2.urlopen('{}/{}'.format(url_base, test_file_name))
    except (urllib2.HTTPError, urllib2.URLError):
        return False
    file_raw = response.read()

    """ check if the test path exist otherwise we create it """
    if not os.path.exists(tests_path):
        os.makedirs(tests_path)

    with open('{}/{}'.format(tests_path, test_file_name), 'w') as f:
        f.write(file_raw)
    return True

def main():
    """ configure script """
    if not (args.test_name in tests):
        logger.error('argument not valid')
        exit(-1)

    if args.test_name == "all":
        for test_name in tests:
            if not (test_name == 'all' or test_name == 'tempest'):
                print(test_name)
                run_task(test_name)
    else:
        print(args.test_name)
        if args.test_name == 'tempest':
            run_tempest()
        else:
            run_task(args.test_name)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import unittest

import cupy
from cupy import cuda
from cupy import testing

@testing.gpu
class TestFromData(unittest.TestCase):

    _multiprocess_can_split_ = True

    @testing.for_all_dtypes()
    @testing.numpy_cupy_array_equal()
    def test_array(self, xp, dtype):
        return xp.array([[1, 2, 3], [2, 3, 4]], dtype=dtype)

    @testing.for_all_dtypes()
    @testing.numpy_cupy_array_equal()
    def test_array_copy(self, xp, dtype):
        a = testing.shaped_arange((2, 3, 4), xp, dtype)
        return xp.array(a)

    @testing.for_all_dtypes()
    @testing.numpy_cupy_array_equal()
    def test_array_copy_is_copied(self, xp, dtype):
        a = testing.shaped_arange((2, 3, 4), xp, dtype)
        b = xp.array(a)
        a.fill(0)
        return b

    @testing.for_all_dtypes(name='dtype1')
    @testing.for_all_dtypes(name='dtype2')
    @testing.numpy_cupy_array_equal()
    def test_array_copy_with_dtype(self, xp, dtype1, dtype2):
        a = testing.shaped_arange((2, 3, 4), xp, dtype1)
        return xp.array(a, dtype=dtype2)

    @testing.for_all_dtypes()
    @testing.numpy_cupy_array_equal()
    def test_asarray(self, xp, dtype):
        a = testing.shaped_arange((2, 3, 4), xp, dtype)
        return xp.asarray(a)

    @testing.for_all_dtypes()
    @testing.numpy_cupy_array_equal()
    def test_asarray_is_not_copied(self, xp, dtype):
        a = testing.shaped_arange((2, 3, 4), xp, dtype)
        b = xp.asarray(a)
        a.fill(0)
        return b

    def test_ascontiguousarray_on_noncontiguous_array(self):
        a = testing.shaped_arange((2, 3, 4))
        b = a.transpose(2, 0, 1)
        c = cupy.ascontiguousarray(b)
        self.assertTrue(c.flags.c_contiguous)
        testing.assert_array_equal(b, c)

    def test_ascontiguousarray_on_contiguous_array(self):
        a = testing.shaped_arange((2, 3, 4))
        b = cupy.ascontiguousarray(a)
        self.assertIs(a, b)

    @testing.for_CF_orders()
    @testing.for_all_dtypes()
    @testing.numpy_cupy_array_equal()
    def test_copy(self, xp, dtype, order):
        a = xp.zeros((2, 3, 4), dtype=dtype)
        b = a.copy(order=order)
        a[1] = 1
        return b

    @testing.multi_gpu(2)
    @testing.for_CF_orders()
    @testing.for_all_dtypes()
    def test_copy_multigpu(self, dtype, order):
        with cuda.Device(0):
            src = cupy.random.uniform(-1, 1, (2, 3)).astype(dtype)
        with cuda.Device(1):
            dst = src.copy(order)
        testing.assert_allclose(src, dst, rtol=0, atol=0)
<commit_msg>Add test for array method<commit_after>import unittest

import cupy
from cupy import cuda
from cupy import testing

@testing.gpu
class TestFromData(unittest.TestCase):

    _multiprocess_can_split_ = True

    @testing.for_all_dtypes()
    @testing.numpy_cupy_array_equal()
    def test_array(self, xp, dtype):
        return xp.array([[1, 2, 3], [2, 3, 4]], dtype=dtype)

    @testing.for_all_dtypes()
    @testing.numpy_cupy_array_equal()
    def test_array_copy(self, xp, dtype):
        a = testing.shaped_arange((2, 3, 4), xp, dtype)
        return xp.array(a)

    @testing.for_all_dtypes()
    @testing.numpy_cupy_array_equal()
    def test_array_copy_is_copied(self, xp, dtype):
        a = testing.shaped_arange((2, 3, 4), xp, dtype)
        b = xp.array(a)
        a.fill(0)
        return b

    @testing.for_all_dtypes(name='dtype1')
    @testing.for_all_dtypes(name='dtype2')
    @testing.numpy_cupy_array_equal()
    def test_array_copy_with_dtype(self, xp, dtype1, dtype2):
        a = testing.shaped_arange((2, 3, 4), xp, dtype1)
        return xp.array(a, dtype=dtype2)

    @testing.for_all_dtypes()
    @testing.numpy_cupy_array_equal()
    def test_array_no_copy(self, xp, dtype):
        a = testing.shaped_arange((2, 3, 4), xp, dtype)
        b = xp.array(a, copy=False)
        a.fill(0)
        return b

    @testing.for_all_dtypes()
    @testing.numpy_cupy_array_equal()
    def test_array_no_copy_ndmin(self, xp, dtype):
        a = testing.shaped_arange((2, 3, 4), xp, dtype)
        b = xp.array(a, copy=False, ndmin=5)
        self.assertEqual(a.shape, (2, 3, 4))
        a.fill(0)
        return b

    @testing.for_all_dtypes()
    @testing.numpy_cupy_array_equal()
    def test_asarray(self, xp, dtype):
        a = testing.shaped_arange((2, 3, 4), xp, dtype)
        return xp.asarray(a)

    @testing.for_all_dtypes()
    @testing.numpy_cupy_array_equal()
    def test_asarray_is_not_copied(self, xp, dtype):
        a = testing.shaped_arange((2, 3, 4), xp, dtype)
        b = xp.asarray(a)
        a.fill(0)
        return b

    def test_ascontiguousarray_on_noncontiguous_array(self):
        a = testing.shaped_arange((2, 3, 4))
        b = a.transpose(2, 0, 1)
        c = cupy.ascontiguousarray(b)
        self.assertTrue(c.flags.c_contiguous)
        testing.assert_array_equal(b, c)

    def test_ascontiguousarray_on_contiguous_array(self):
        a = testing.shaped_arange((2, 3, 4))
        b = cupy.ascontiguousarray(a)
        self.assertIs(a, b)

    @testing.for_CF_orders()
    @testing.for_all_dtypes()
    @testing.numpy_cupy_array_equal()
    def test_copy(self, xp, dtype, order):
        a = xp.zeros((2, 3, 4), dtype=dtype)
        b = a.copy(order=order)
        a[1] = 1
        return b

    @testing.multi_gpu(2)
    @testing.for_CF_orders()
    @testing.for_all_dtypes()
    def test_copy_multigpu(self, dtype, order):
        with cuda.Device(0):
            src = cupy.random.uniform(-1, 1, (2, 3)).astype(dtype)
        with cuda.Device(1):
            dst = src.copy(order)
        testing.assert_allclose(src, dst, rtol=0, atol=0)
<|endoftext|>


<commit_before>"""
Tests for the display interface (display chrome extension).

"""

from functools import partial

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as exp_cond

from base import TestBase
from base import MAPS_URL
from base import screenshot_on_error

# these particular elements are present when the display extension is not loaded
# some are IDs, some are class names - as stated lookup method
elements = (("zoom", By.ID),
            ("widget-zoom-button", By.CLASS_NAME),
            ("widget-zoom-in", By.CLASS_NAME),
            ("widget-zoom-out", By.CLASS_NAME),
            ("compass", By.ID),
            ("searchboxinput", By.ID),
            ("searchbox_form", By.ID),
            ("searchbutton", By.CLASS_NAME),
            ("acme-poi-button", By.CLASS_NAME),
            ("acme-points-of-interest", By.ID),
            ("widget-runway-tray-wrapper", By.CLASS_NAME),
            ("acme-famous-places-button", By.ID),
            ("widget-runway-thumbstrip-background", By.CLASS_NAME))

class TestBaseDisplay(TestBase):
    """
    Loads default extensions for display,
    all tests for display should inherit from this class.

    Test that the following graphical components are not displayed:
    zoom in/out buttons
    compass
    google More Fun menu
    search field
    search button
    runway panel

    """
    extensions = ["display", ]

    @screenshot_on_error
    def test_widgets_not_displayed(self):
        """
        After the browser loads the URL, it takes some time until
        also the extensions are fully loaded and the modify already
        loaded DOM. E.g. the graphical widgets (elements) in question
        are removed.

        """
        config = self.get_config()
        self.browser.get(MAPS_URL)

        def test_elements_not_present(elem):
            msg = "Element '{0}' should not be present (extension display).".format(elem[0])
            # An Expectation for checking that an element is either
            # invisible or not present on the DOM.
            tester = partial(exp_cond.invisibility_of_element_located, (elem[1], elem[0]))
            WebDriverWait(self.browser,
                          config["max_load_timeout"]).until(tester(), message=msg)
        map(test_elements_not_present, elements)

class TestBaseKioskExtension(TestBase):
    """
    Test DOM elements are displayed with kiosk extension
    against them not displayed with the display extension.

    """
    extensions = ["kiosk", ]

    @screenshot_on_error
    def test_widgets_displayed(self):
        """
        If elements from the elements list are renamed, the test
        test_widgets_not_displayed() would just test that they are not present,
        however they might be, but under a different name.
        Test that exactly these elements are present when the display
        extension is not loaded.

        """
        config = self.get_config()
        self.browser.get(MAPS_URL)

        def test_elements_present(elem):
            msg = "Element '{0}' should be present (kiosk extension).".format(elem[0])
            tester = partial(exp_cond.presence_of_element_located, (elem[1], elem[0]))
            WebDriverWait(self.browser,
                          config["max_load_timeout"]).until(tester(), message=msg)
        map(test_elements_present, elements)
<commit_msg>enforce screenshot in test_display, compass issue<commit_after>"""
Tests for the display interface (display chrome extension).

"""

from functools import partial

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as exp_cond

from base import TestBase
from base import MAPS_URL
from base import screenshot_on_error, make_screenshot

# these particular elements are present when the display extension is not loaded
# some are IDs, some are class names - as stated lookup method
elements = (("zoom", By.ID),
            ("widget-zoom-button", By.CLASS_NAME),
            ("widget-zoom-in", By.CLASS_NAME),
            ("widget-zoom-out", By.CLASS_NAME),
            ("compass", By.ID),
            ("searchboxinput", By.ID),
            ("searchbox_form", By.ID),
            ("searchbutton", By.CLASS_NAME),
            ("acme-poi-button", By.CLASS_NAME),
            ("acme-points-of-interest", By.ID),
            ("widget-runway-tray-wrapper", By.CLASS_NAME),
            ("acme-famous-places-button", By.ID),
            ("widget-runway-thumbstrip-background", By.CLASS_NAME))

class TestBaseDisplay(TestBase):
    """
    Loads default extensions for display,
    all tests for display should inherit from this class.

    Test that the following graphical components are not displayed:
    zoom in/out buttons
    compass
    google More Fun menu
    search field
    search button
    runway panel

    """
    extensions = ["display", ]

    @screenshot_on_error
    def test_widgets_not_displayed(self):
        """
        After the browser loads the URL, it takes some time until
        also the extensions are fully loaded and the modify already
        loaded DOM. E.g. the graphical widgets (elements) in question
        are removed.

        """
        config = self.get_config()
        self.browser.get(MAPS_URL)

        def test_elements_not_present(elem):
            msg = "Element '{0}' should not be present (extension display).".format(elem[0])
            # An Expectation for checking that an element is either
            # invisible or not present on the DOM.
            tester = partial(exp_cond.invisibility_of_element_located, (elem[1], elem[0]))
            WebDriverWait(self.browser,
                          config["max_load_timeout"]).until(tester(), message=msg)
        map(test_elements_not_present, elements)
        make_screenshot(self.browser, "test_elements_not_present", 0)

class TestBaseKioskExtension(TestBase):
    """
    Test DOM elements are displayed with kiosk extension
    against them not displayed with the display extension.

    """
    extensions = ["kiosk", ]

    @screenshot_on_error
    def test_widgets_displayed(self):
        """
        If elements from the elements list are renamed, the test
        test_widgets_not_displayed() would just test that they are not present,
        however they might be, but under a different name.
        Test that exactly these elements are present when the display
        extension is not loaded.

        """
        config = self.get_config()
        self.browser.get(MAPS_URL)

        def test_elements_present(elem):
            msg = "Element '{0}' should be present (kiosk extension).".format(elem[0])
            tester = partial(exp_cond.presence_of_element_located, (elem[1], elem[0]))
            WebDriverWait(self.browser,
                          config["max_load_timeout"]).until(tester(), message=msg)
        map(test_elements_present, elements)
        make_screenshot(self.browser, "test_widgets_displayed", 0)<|endoftext|>