<commit_before>#!/usr/bin/env python
#
# Mod Tee functionnal tests
#
# To use this script, see ./run.py -h

import os, glob, time
import cStringIO
import optparse
import pycurl
import BaseHTTPServer
import multiprocessing
import Queue
import re
import sys
import urllib

class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):

    def do_GET(self):
        post_body = ''.join(iter(self.rfile.read, ''))
        self.server.queue.put(('POST', self.path, post_body, self.server.server_port))
        # FIXME: why is the pipe broken at this point?
        #self.send_response(200)

    def do_POST(self):
        post_body = ''.join(iter(self.rfile.read, ''))
        self.server.queue.put(('GET', self.path, post_body, self.server.server_port))
        # FIXME: why is the pipe broken at this point?
        #self.send_response(200)

def http_server(q, host, port):
    server = BaseHTTPServer.HTTPServer((host, port), RequestHandler)
    server.queue = q
    server.serve_forever()

class TeeRequest:
    def __init__(self, filename, host, port):
        self.filename = filename
        self.host, self.port = host, port
        self.parse()

    def parse(self):
        """
        Parses the request file
        """
        f = open(self.filename, 'r')
        _ = self.consume(f, '==DESC==')
        self.desc = self.consume(f, "==HEADER==")
        self.path = self.consume(f, "==BODY==")
        self.body = self.consume(f, "==THEADER==")
        self.t_path = self.consume(f, "==TBODY==")
        self.t_body = self.consume(f, "==RBODY==")
        self.r_body = self.consume(f, "==DEST==")
        self.t_dest = self.consume(f, "==EOF==")
        f.close()

    def __str__(self):
        return '''Request\n
  desc: %s
  path: %s
  body: %s
  t_path: %s
  t_body: %s''' % (self.desc, self.path, self.body, self.t_path, self.t_body)

    def consume(self, f, key=''):
        lines = []
        while 1:
            line = f.readline()
            if line == '':
                line = '==EOF=='
                if key != line:
                    raise Exception('EOF while trying to consume %s' % key)
            line = line.strip()
            if line == key:
                break
            lines.append(line)
        return ''.join(lines)

    def get_url(self):
        return "http://%s:%d%s" % (self.host, self.port, self.path)

    def play_with_curl(self, curl, verbose=False):
        buf = cStringIO.StringIO()

        if verbose:
            print "Url:", self.get_url()
        curl.setopt(curl.URL, self.get_url())
        curl.setopt(curl.WRITEFUNCTION, buf.write)
        curl.setopt(curl.FOLLOWLOCATION, 1);
        if (len(self.body)):
            curl.setopt(curl.POST, 1)
            curl.setopt(curl.HTTPHEADER, ['Content-Type: text/xml; charset=utf-8',
                                          'Content-Length: ' + str(len(self.body))] )
            curl.setopt(curl.POSTFIELDSIZE, len(self.body))
            curl.setopt(curl.POSTFIELDS, self.body)
        assert not curl.perform()
        return buf

    def assert_received(self, path, body, server_port):
        # if (self.t_dest == "MULTI")
        if (len(self.t_dest) and self.t_dest != "MULTI"):
            assert server_port == 16555 ,  "########### SHOULD BE ON SECOND LOCATION  ###############"
        elif not len(self.t_dest):
            assert  server_port != 16555 ,  "########### SHOULD BE ON FIRST LOCATION  ###############"
        assert self.t_path, '''Unexpected request received
               path: %s
               body: %s''' % (path, body)
        assert self.t_path and path == self.t_path ,\
                '''Path did not match:
                   path: %s
                   body: %s''' % (path, body)
        assert re.search(self.t_body, body, re.MULTILINE|re.DOTALL),\
                 '''Body did not match:
                 path: %s
                 body: %s''' % (path, body)

    def assert_not_received(self):
        assert not self.t_path and not self.t_body, 'Request not duplicated'

def run_tests(request_files, queue, options):

    for r_fname in request_files:
        print "Test:", r_fname
        curl = pycurl.Curl()
        request = TeeRequest(r_fname, options.host, int(options.port))
        response = request.play_with_curl(curl, verbose=options.verbose)

        if (len(request.r_body)):
            assert request.r_body == response.getvalue().rstrip(), '''Response mismatch:
   expected: %s
   received: %s''' % (request.r_body, response.getvalue())
        if not options.curl_only:
            try:
                try:
                    method, path, body, server_port = queue.get(timeout=3)
                    request.assert_received(path, body, server_port)
                    if (request.t_dest == "MULTI"):
                        # second extraction from the queue
                        method2, path2, body2, server_port2 = queue.get(timeout=3)
                        assert server_port != server_port2, "Multi sent on the same location"
                        request.assert_received(path2, body2, server_port2)

                except Queue.Empty:
                    request.assert_not_received()

            except AssertionError, err:
                print "########### RECEIVED ###############"
                print "Error:", err
                print "************************************"
                print "########### EXPECTED ###############"
                print "Expected:", request
                exit(1)

def display_test_content(request_files):
    for r_fname in request_files:
        request = TeeRequest(r_fname, '', int(8000))
        print ("Test: %s " % r_fname + request.desc)

def main(options, args):
    # Stats the request files
    request_file_pattern = os.path.join(options.path, options.test)
    request_files = glob.glob(request_file_pattern)
    request_files.sort()
    if not request_files:
        sys.stdout.write ("No files matching %s" % request_file_pattern)
    else:

        if options.desc:
            display_test_content(request_files)
            exit(0)
        process = None
        queue = None
        try:
            if not options.curl_only:
                queue = multiprocessing.Queue()
                ## First process
                process = multiprocessing.Process(target=http_server, args=(queue, options.dest_host, options.dest_port))
                process.start()
                ## Second process on port + 1
                process2 = multiprocessing.Process(target=http_server, args=(queue, options.dest_host, 16555))
                process2.start()
                # We need to wait for the server to start up...
                time.sleep(2)
            run_tests(request_files, queue, options)
        except Exception as err_msg:
            print err_msg
            exit(0)
        finally:
            if not options.curl_only:
                process.terminate()
                process2.terminate()
                process.join()
                process2.join()

if __name__ == '__main__':
    # By default, we use the test files in the data folder next to this script
    default_path = os.path.join(os.getcwd(), os.path.dirname(__file__), "data")

    # Option parsing
    arg_parser = optparse.OptionParser()
    arg_parser.add_option('--test', dest='test',
                        help='Pattern of test file(s) to run', default='*.req')
    arg_parser.add_option('--curl', dest='curl_only', action='store_true',
                        help='Only plays the request with curl', default=False)
    arg_parser.add_option('--verbose', dest='verbose', action='store_true',
                        help='More verbose output', default=False)
    arg_parser.add_option('--server', dest='host',
                        help='The server to send the requests to', default="localhost")
    arg_parser.add_option('--port', dest='port',
                        help='The port to send the requests to', default=8042)
    arg_parser.add_option('--desc', dest='desc',
                        help='Display the tests actions and exit', default=False)
    arg_parser.add_option('--dest_server', dest='dest_host',
                        help='The destination server of mod_tee.', default="localhost")
    arg_parser.add_option('--dest_port', dest='dest_port',
                        help='The destination port of mod_tee', default=8043)
    arg_parser.add_option('--path', dest='path',
                        help='The folder containing the request files', default=default_path)

    main(*arg_parser.parse_args())
<commit_msg>Improved script test_dup to handle dup header assertions<commit_after>#!/usr/bin/env python
#
# Mod Tee functionnal tests
#
# To use this script, see ./run.py -h

import os, glob, time
import cStringIO
import optparse
import pycurl
import BaseHTTPServer
import multiprocessing
import Queue
import re
import sys
import urllib

class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):

    def do_GET(self):
        posdup_body = ''.join(iter(self.rfile.read, ''))
        self.server.queue.put(('POST', self.path, posdup_body, self.server.server_port))
        # FIXME: why is the pipe broken at this point?
        #self.send_response(200)

    def do_POST(self):
        posdup_body = ''.join(iter(self.rfile.read, ''))
        self.server.queue.put(('GET', self.path, posdup_body, self.server.server_port))
        # FIXME: why is the pipe broken at this point?
        #self.send_response(200)

def http_server(q, host, port):
    server = BaseHTTPServer.HTTPServer((host, port), RequestHandler)
    server.queue = q
    server.serve_forever()

class DupRequest:
    def __init__(self, filename, host, port):
        self.filename = filename
        self.host, self.port = host, port
        self.parse()

    def parse(self):
        """
        Parses the request file
        """
        f = open(self.filename, 'r')
        _ = self.consume(f, '==DESC==')
        self.desc = self.consume(f, "==URL==")
        self.path = self.consume(f, "==BODY==")
        self.body = self.consume(f, "==DUPURL==")
        self.dup_path = self.consume(f, "==DUPHEADER==")
        self.dup_header = self.consumeIntoList(f, "==DUPBODY==")
        self.dup_body = self.consume(f, "==RESPBODY==")
        self.resp_body = self.consume(f, "==DUPDESTPORT==")
        self.dup_dest = self.consume(f, "==EOF==")
        f.close()
        # actual dup HTTP headers (ones received from mod_dup)
        self.response_dup_header = list()
        self.response_dup_body = cStringIO.StringIO()

    def __str__(self):
        return '''Request\n
  desc: %s
  path: %s
  body: %s
  dup_path: %s
  dup_body: %s''' % (self.desc, self.path, self.body, self.dup_path, self.dup_body)

    ## RETURN CONCATENATION OF ALL THE LINES CONSUMED
    def consume(self, f, key=''):
        lines = []
        while 1:
            line = f.readline()
            if line == '':
                line = '==EOF=='
                if key != line:
                    raise Exception('EOF while trying to consume %s' % key)
            line = line.strip()
            if line == key:
                break
            lines.append(line)
        return ''.join(lines)

    ## RETURN ALL THE LINES CONSUMED AS A LIST
    def consumeIntoList(self, f, key=''):
        lines = []
        while 1:
            line = f.readline()
            if line == '':
                line = '==EOF=='
                if key != line:
                    raise Exception('EOF while trying to consume %s' % key)
            line = line.strip()
            if line == key:
                break
            lines.append(line)
        return lines

    def get_url(self):
        return "http://%s:%d%s" % (self.host, self.port, self.path)

    def header_handler(self, header_line):
        header_line = header_line.decode('iso-8859-1')
        if ':' not in header_line:
            return
        self.response_dup_header.append(str(header_line).rstrip())

    ## EXECUTE THE CURL REQUEST
    def execute(self, curl, verbose=False):
        if verbose:
            print "Url:", self.get_url()
        curl.setopt(curl.URL, self.get_url())
        curl.setopt(curl.HEADERFUNCTION, self.header_handler) # pycurl executes header_handler on each line of the http header
        curl.setopt(curl.WRITEFUNCTION, self.response_dup_body.write) #pycurl executes write on self.response_dup_body for the body
        curl.setopt(curl.FOLLOWLOCATION, 1);
        if (len(self.body)):
            curl.setopt(curl.POST, 1)
            curl.setopt(curl.HTTPHEADER, ['Content-Type: text/xml; charset=utf-8',
                                          'Content-Length: ' + str(len(self.body))] )
            curl.setopt(curl.POSTFIELDSIZE, len(self.body))
            curl.setopt(curl.POSTFIELDS, self.body)
        assert not curl.perform()
        return self.response_dup_body

    ## MAKE THE ASSERTIONS/COMPARISONS
    def assert_received(self, path, body, server_port):
        # if (self.dup_dest == "MULTI")
        if (len(self.dup_dest) and self.dup_dest != "MULTI"):
            assert server_port == 16555 ,  "########### SHOULD BE ON SECOND LOCATION  ###############"
        elif not len(self.dup_dest):
            assert  server_port != 16555 ,  "########### SHOULD BE ON FIRST LOCATION  ###############"
        assert self.dup_path, '''Unexpected request received
               path: %s
               body: %s''' % (path, body)
        assert self.dup_path and path == self.dup_path ,\
                '''Path did not match:
                   path: %s
                   body: %s''' % (path, body)
        assert re.search(self.dup_body, body, re.MULTILINE|re.DOTALL),\
                 '''Body did not match:
                 path: %s
                 body: %s''' % (path, body)
        # check headers
        for headerLineExpected in self.dup_header: # iterate through expected header lines (defined by ==DUPHEADER==)
            contained = False
            for headerLineActual in self.response_dup_header: # iterate through received header lines
                if re.search(headerLineExpected,headerLineActual) != None:
                    contained = True # header line is contained, OK
                    break
            assert contained,\
            '''Header not found: %s''' % (headerLineExpected)

    def assert_not_received(self):
        assert not self.dup_path and not self.dup_body, 'Request not duplicated'

def run_tests(request_files, queue, options):

    for r_fname in request_files:
        print "Test:", r_fname
        curl = pycurl.Curl()
        request = DupRequest(r_fname, options.host, int(options.port))
        request.execute(curl, verbose=options.verbose)

        if (len(request.resp_body)):
            assert request.resp_body == request.response_dup_body.getvalue().rstrip(), '''Response mismatch:
   expected: %s
   received: %s''' % (request.resp_body, request.response_dup_body.getvalue())
        if not options.curl_only:
            try:
                try:
                    method, path, body, server_port = queue.get(timeout=3)
                    request.assert_received(path, body, server_port)
                    if (request.dup_dest == "MULTI"):
                        # second extraction from the queue
                        method2, path2, body2, server_port2 = queue.get(timeout=3)
                        assert server_port != server_port2, "Multi sent on the same location"
                        request.assert_received(path2, body2, server_port2)

                except Queue.Empty:
                    request.assert_not_received()

            except AssertionError, err:
                print "########### RECEIVED ###############"
                print "Error:", err
                print "************************************"
                print "########### EXPECTED ###############"
                print "Expected:", request
                exit(1)

def display_test_content(request_files):
    for r_fname in request_files:
        request = DupRequest(r_fname, '', int(8000))
        print ("Test: %s " % r_fname + request.desc)

def main(options, args):
    # Stats the request files
    request_file_pattern = os.path.join(options.path, options.test)
    request_files = glob.glob(request_file_pattern)
    request_files.sort()
    if not request_files:
        sys.stdout.write ("No files matching %s" % request_file_pattern)
    else:

        if options.desc:
            display_test_content(request_files)
            exit(0)
        process = None
        queue = None
        try:
            if not options.curl_only:
                queue = multiprocessing.Queue()
                ## First process
                process = multiprocessing.Process(target=http_server, args=(queue, options.dest_host, options.dest_port))
                process.start()
                ## Second process on port + 1
                process2 = multiprocessing.Process(target=http_server, args=(queue, options.dest_host, 16555))
                process2.start()
                # We need to wait for the server to start up...
                time.sleep(2)
            run_tests(request_files, queue, options)
        except Exception as err_msg:
            print err_msg
            exit(0)
        finally:
            if not options.curl_only:
                process.terminate()
                process2.terminate()
                process.join()
                process2.join()

if __name__ == '__main__':
    # By default, we use the test files in the data folder next to this script
    defauldup_path = os.path.join(os.getcwd(), os.path.dirname(__file__), "data")

    # Option parsing
    arg_parser = optparse.OptionParser()
    arg_parser.add_option('--test', dest='test',
                        help='Pattern of test file(s) to run', default='*.req')
    arg_parser.add_option('--curl', dest='curl_only', action='store_true',
                        help='Only plays the request with curl', default=False)
    arg_parser.add_option('--verbose', dest='verbose', action='store_true',
                        help='More verbose output', default=False)
    arg_parser.add_option('--server', dest='host',
                        help='The server to send the requests to', default="localhost")
    arg_parser.add_option('--port', dest='port',
                        help='The port to send the requests to', default=8042)
    arg_parser.add_option('--desc', dest='desc',
                        help='Display the tests actions and exit', default=False)
    arg_parser.add_option('--dest_server', dest='dest_host',
                        help='The destination server of mod_tee.', default="localhost")
    arg_parser.add_option('--dest_port', dest='dest_port',
                        help='The destination port of mod_tee', default=8043)
    arg_parser.add_option('--path', dest='path',
                        help='The folder containing the request files', default=defauldup_path)

    main(*arg_parser.parse_args())
<|endoftext|>


<commit_before># Analyzes an object and outputs numeric properties

import cv2
import numpy as np
import os
from plantcv.plantcv import fatal_error
from plantcv.plantcv import print_image
from plantcv.plantcv import plot_image
from plantcv.plantcv import rgb2gray_hsv
from plantcv.plantcv import find_objects
from plantcv.plantcv.threshold import binary as binary_threshold
from plantcv.plantcv import roi_objects
from plantcv.plantcv import object_composition
from plantcv.plantcv import apply_mask
from plantcv.plantcv import params
from plantcv.plantcv import outputs

def report_size_marker_area(img, roi_contour, roi_hierarchy, marker='define', objcolor='dark', thresh_channel=None,
                            thresh=None):
    """Detects a size marker in a specified region and reports its size and eccentricity

    Inputs:
    img             = An RGB or grayscale image to plot the marker object on
    roi_contour     = A region of interest contour (e.g. output from pcv.roi.rectangle or other methods)
    roi_hierarchy   = A region of interest contour hierarchy (e.g. output from pcv.roi.rectangle or other methods)
    marker          = 'define' or 'detect'. If define it means you set an area, if detect it means you want to
                      detect within an area
    objcolor        = Object color is 'dark' or 'light' (is the marker darker or lighter than the background)
    thresh_channel  = 'h', 's', or 'v' for hue, saturation or value
    thresh          = Binary threshold value (integer)

    Returns:
    analysis_images = List of output images

    :param img: numpy.ndarray
    :param roi_contour: list
    :param roi_hierarchy: numpy.ndarray
    :param marker: str
    :param objcolor: str
    :param thresh_channel: str
    :param thresh: int
    :return: analysis_images: list
    """
    # Store debug
    debug = params.debug
    params.debug = None

    params.device += 1
    # Make a copy of the reference image
    ref_img = np.copy(img)
    # If the reference image is grayscale convert it to color
    if len(np.shape(ref_img)) == 2:
        ref_img = cv2.cvtColor(ref_img, cv2.COLOR_GRAY2BGR)

    # Marker components
    # If the marker type is "defined" then the marker_mask and marker_contours are equal to the input ROI
    # Initialize a binary image
    roi_mask = np.zeros(np.shape(img)[:2], dtype=np.uint8)
    # Draw the filled ROI on the mask
    cv2.drawContours(roi_mask, roi_contour, -1, (255), -1)
    marker_mask = []
    marker_contour = []

    # If the marker type is "detect" then we will use the ROI to isolate marker contours from the input image
    if marker.upper() == 'DETECT':
        # We need to convert the input image into an one of the HSV channels and then threshold it
        if thresh_channel is not None and thresh is not None:
            # Mask the input image
            masked = apply_mask(img=ref_img, mask=roi_mask, mask_color="black")
            # Convert the masked image to hue, saturation, or value
            marker_hsv = rgb2gray_hsv(rgb_img=masked, channel=thresh_channel)
            # Threshold the HSV image
            marker_bin = binary_threshold(gray_img=marker_hsv, threshold=thresh, max_value=255, object_type=objcolor)
            # Identify contours in the masked image
            contours, hierarchy = find_objects(img=ref_img, mask=marker_bin)
            # Filter marker contours using the input ROI
            kept_contours, kept_hierarchy, kept_mask, obj_area = roi_objects(img=ref_img, object_contour=contours,
                                                                             obj_hierarchy=hierarchy,
                                                                             roi_contour=roi_contour,
                                                                             roi_hierarchy=roi_hierarchy,
                                                                             roi_type="partial")
            # If there are more than one contour detected, combine them into one
            # These become the marker contour and mask
            marker_contour, marker_mask = object_composition(img=ref_img, contours=kept_contours,
                                                             hierarchy=kept_hierarchy)
        else:
            # Reset debug mode
            params.debug = debug
            fatal_error('thresh_channel and thresh must be defined in detect mode')
    elif marker.upper() == "DEFINE":
        # Identify contours in the masked image
        contours, hierarchy = find_objects(img=ref_img, mask=roi_mask)
        # If there are more than one contour detected, combine them into one
        # These become the marker contour and mask
        marker_contour, marker_mask = object_composition(img=ref_img, contours=contours, hierarchy=hierarchy)
    else:
        # Reset debug mode
        params.debug = debug
        fatal_error("marker must be either 'define' or 'detect' but {0} was provided.".format(marker))

    # Calculate the moments of the defined marker region
    m = cv2.moments(marker_mask, binaryImage=True)
    # Calculate the marker area
    marker_area = m['m00']

    # Fit a bounding ellipse to the marker
    center, axes, angle = cv2.fitEllipse(marker_contour)
    major_axis = np.argmax(axes)
    minor_axis = 1 - major_axis
    major_axis_length = axes[major_axis]
    minor_axis_length = axes[minor_axis]
    # Calculate the bounding ellipse eccentricity
    eccentricity = np.sqrt(1 - (axes[minor_axis] / axes[major_axis]) ** 2)

    cv2.drawContours(ref_img, marker_contour, -1, (255, 0, 0), 5)
    analysis_image = ref_img

    # Reset debug mode
    params.debug = debug

    if params.debug == 'print':
        print_image(ref_img, os.path.join(params.debug_outdir, str(params.device) + '_marker_shape.png'))
    elif params.debug == 'plot':
        plot_image(ref_img)

    outputs.add_observation(variable='marker_area', trait='marker area',
                            method='plantcv.plantcv.report_size_marker_area', scale='pixels', datatype=int,
                            value=marker_area, label='pixels')
    outputs.add_observation(variable='marker_ellipse_major_axis', trait='marker ellipse major axis length',
                            method='plantcv.plantcv.report_size_marker_area', scale='pixels', datatype=int,
                            value=major_axis_length, label='pixels')
    outputs.add_observation(variable='marker_ellipse_minor_axis', trait='marker ellipse minor axis length',
                            method='plantcv.plantcv.report_size_marker_area', scale='pixels', datatype=int,
                            value=minor_axis_length, label='pixels')
    outputs.add_observation(variable='marker_ellipse_eccentricity', trait='marker ellipse eccentricity',
                            method='plantcv.plantcv.report_size_marker_area', scale='none', datatype=float,
                            value=eccentricity, label='none')

    # Store images
    outputs.images.append(analysis_image)

    return analysis_image
<commit_msg>Update report_size_marker_area.py<commit_after># Analyzes an object and outputs numeric properties

import cv2
import numpy as np
import os
from plantcv.plantcv import fatal_error
from plantcv.plantcv import print_image
from plantcv.plantcv import plot_image
from plantcv.plantcv import rgb2gray_hsv
from plantcv.plantcv import find_objects
from plantcv.plantcv.threshold import binary as binary_threshold
from plantcv.plantcv import roi_objects
from plantcv.plantcv import object_composition
from plantcv.plantcv import apply_mask
from plantcv.plantcv import params
from plantcv.plantcv import outputs

def report_size_marker_area(img, roi_contour, roi_hierarchy, marker='define', objcolor='dark', thresh_channel=None,
                            thresh=None, label=None):
    """Detects a size marker in a specified region and reports its size and eccentricity

    Inputs:
    img             = An RGB or grayscale image to plot the marker object on
    roi_contour     = A region of interest contour (e.g. output from pcv.roi.rectangle or other methods)
    roi_hierarchy   = A region of interest contour hierarchy (e.g. output from pcv.roi.rectangle or other methods)
    marker          = 'define' or 'detect'. If define it means you set an area, if detect it means you want to
                      detect within an area
    objcolor        = Object color is 'dark' or 'light' (is the marker darker or lighter than the background)
    thresh_channel  = 'h', 's', or 'v' for hue, saturation or value
    thresh          = Binary threshold value (integer)
    label      = optional label parameter, modifies the variable name of observations recorded

    Returns:
    analysis_images = List of output images

    :param img: numpy.ndarray
    :param roi_contour: list
    :param roi_hierarchy: numpy.ndarray
    :param marker: str
    :param objcolor: str
    :param thresh_channel: str
    :param thresh: int
    :param label: str
    :return: analysis_images: list
    """
    # Store debug
    debug = params.debug
    params.debug = None

    params.device += 1
    # Make a copy of the reference image
    ref_img = np.copy(img)
    # If the reference image is grayscale convert it to color
    if len(np.shape(ref_img)) == 2:
        ref_img = cv2.cvtColor(ref_img, cv2.COLOR_GRAY2BGR)

    # Marker components
    # If the marker type is "defined" then the marker_mask and marker_contours are equal to the input ROI
    # Initialize a binary image
    roi_mask = np.zeros(np.shape(img)[:2], dtype=np.uint8)
    # Draw the filled ROI on the mask
    cv2.drawContours(roi_mask, roi_contour, -1, (255), -1)
    marker_mask = []
    marker_contour = []

    # If the marker type is "detect" then we will use the ROI to isolate marker contours from the input image
    if marker.upper() == 'DETECT':
        # We need to convert the input image into an one of the HSV channels and then threshold it
        if thresh_channel is not None and thresh is not None:
            # Mask the input image
            masked = apply_mask(img=ref_img, mask=roi_mask, mask_color="black")
            # Convert the masked image to hue, saturation, or value
            marker_hsv = rgb2gray_hsv(rgb_img=masked, channel=thresh_channel)
            # Threshold the HSV image
            marker_bin = binary_threshold(gray_img=marker_hsv, threshold=thresh, max_value=255, object_type=objcolor)
            # Identify contours in the masked image
            contours, hierarchy = find_objects(img=ref_img, mask=marker_bin)
            # Filter marker contours using the input ROI
            kept_contours, kept_hierarchy, kept_mask, obj_area = roi_objects(img=ref_img, object_contour=contours,
                                                                             obj_hierarchy=hierarchy,
                                                                             roi_contour=roi_contour,
                                                                             roi_hierarchy=roi_hierarchy,
                                                                             roi_type="partial")
            # If there are more than one contour detected, combine them into one
            # These become the marker contour and mask
            marker_contour, marker_mask = object_composition(img=ref_img, contours=kept_contours,
                                                             hierarchy=kept_hierarchy)
        else:
            # Reset debug mode
            params.debug = debug
            fatal_error('thresh_channel and thresh must be defined in detect mode')
    elif marker.upper() == "DEFINE":
        # Identify contours in the masked image
        contours, hierarchy = find_objects(img=ref_img, mask=roi_mask)
        # If there are more than one contour detected, combine them into one
        # These become the marker contour and mask
        marker_contour, marker_mask = object_composition(img=ref_img, contours=contours, hierarchy=hierarchy)
    else:
        # Reset debug mode
        params.debug = debug
        fatal_error("marker must be either 'define' or 'detect' but {0} was provided.".format(marker))

    # Calculate the moments of the defined marker region
    m = cv2.moments(marker_mask, binaryImage=True)
    # Calculate the marker area
    marker_area = m['m00']

    # Fit a bounding ellipse to the marker
    center, axes, angle = cv2.fitEllipse(marker_contour)
    major_axis = np.argmax(axes)
    minor_axis = 1 - major_axis
    major_axis_length = axes[major_axis]
    minor_axis_length = axes[minor_axis]
    # Calculate the bounding ellipse eccentricity
    eccentricity = np.sqrt(1 - (axes[minor_axis] / axes[major_axis]) ** 2)

    cv2.drawContours(ref_img, marker_contour, -1, (255, 0, 0), 5)
    analysis_image = ref_img

    # Reset debug mode
    params.debug = debug

    if label == None:
        prefix = ""
    else:
        prefix = label + "_"

    if params.debug == 'print':
        print_image(ref_img, os.path.join(params.debug_outdir, str(params.device) + prefix + '_marker_shape.png'))
    elif params.debug == 'plot':
        plot_image(ref_img)

    outputs.add_observation(variable=prefix + 'marker_area', trait='marker area',
                            method='plantcv.plantcv.report_size_marker_area', scale='pixels', datatype=int,
                            value=marker_area, label='pixels')
    outputs.add_observation(variable=prefix + 'marker_ellipse_major_axis', trait='marker ellipse major axis length',
                            method='plantcv.plantcv.report_size_marker_area', scale='pixels', datatype=int,
                            value=major_axis_length, label='pixels')
    outputs.add_observation(variable=prefix + 'marker_ellipse_minor_axis', trait='marker ellipse minor axis length',
                            method='plantcv.plantcv.report_size_marker_area', scale='pixels', datatype=int,
                            value=minor_axis_length, label='pixels')
    outputs.add_observation(variable=prefix + 'marker_ellipse_eccentricity', trait='marker ellipse eccentricity',
                            method='plantcv.plantcv.report_size_marker_area', scale='none', datatype=float,
                            value=eccentricity, label='none')

    # Store images
    outputs.images.append(analysis_image)

    return analysis_image
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from werkzeug.contrib.profiler import ProfilerMiddleware
from flask import Flask, request, g, render_template
from flask.ext.triangle import Triangle
from scipy import sparse, io
from sklearn.metrics.pairwise import pairwise_distances
import numpy as np
from matlab import engine
import os, json, time
from flask.ext.cors import CORS
import pdb
from jinja2 import Environment

# Configuration
app = Flask(__name__, static_path='/static')
Triangle(app)
CORS(app)
app.config['CORS_HEADERS'] = 'Content-Type'

app.config['DEBUG'] = True
app.config.from_object(__name__)
app.config.from_envvar('FLASKR_SETTINGS', silent=True)

app.config['PROFILE'] = True
app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[10])
# @app.after_request
# def after_request(response):
#   response.headers.add('Access-Control-Allow-Origin', '*')
#   response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
#   response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE')
#   return response

def supervisedTSNE(distanceMatrix, cl_idx, sameTopicWeight=0.9, differentTopicWeight=1.1):
	n = distanceMatrix.shape[0]
	for i in xrange(n):
		i_topic = cl_idx[i]
		for j in xrange(n):
			if i==j:
				continue
			elif i_topic==cl_idx[j]:
				distanceMatrix[i,j]*=sameTopicWeight
			else:
				distanceMatrix[i,j]*=differentTopicWeight
		distanceMatrix[i,:] /= distanceMatrix[i,:].max()

	return np.round(distanceMatrix,decimals=4)

# Routing
# @app.before_first_request
def before__first_request_():
	global eng
	global mappedX
	global cl_idx
	global Wtopk
	global voca
	global distanceMatrix
	global distanceMatrix_main

	tic = time.time()
	print 'Starting matlab - ',
	eng = engine.start_matlab()
	eng.cd(os.path.dirname(os.getcwd()))
	print "%.4f" % (time.time()-tic)

	#tic = time.time()
	##print "Get data - ",
	#[mappedX, cl_idx, Wtopk_idx,voca] = eng.main_topic(nargout=4)
	#print "%.4f" % (time.time()-tic)

	tic = time.time()
	eng.main_topic2(nargout=0)

	mappedX_ = eng.workspace['mappedX']
	cl_idx_ = eng.workspace['cl_idx']
	Wtopk_idx_ = eng.workspace['Wtopk_idx']
	voca_ = eng.workspace['dict']
	print "%.4f" % (time.time()-tic)

	distanceMatrix = io.loadmat('./../tdm2.mat')['DD']

	tic = time.time()
	print "Calculate data - ",

	Wtopk = []
	for idxArray in Wtopk_idx:
		tempArray = []
		for idx in idxArray:
			tempArray.append(voca[int(idx)-1])
		Wtopk.append(tempArray)

	cl_idx = cl_idx[0]

	cl_idx = cl_idx
	distanceMatrix = distanceMatrix

	cl_idx = cl_idx
	distanceMatrix = distanceMatrix

	sameTopicWeight = 0.9
	differentTopicWeight = 1.1
	distanceMatrix_main = supervisedTSNE(distanceMatrix, cl_idx,
		sameTopicWeight=sameTopicWeight, differentTopicWeight=differentTopicWeight)

	print "%.4f" % (time.time()-tic)

	distanceMatrix_main = distanceMatrix_main.tolist()

@app.teardown_request
def teardown_request(exception):
	print('Teardown arose!'.format(exception))

@app.route('/get_subTopic')
def get_subTopic():
	global eng
	global voca
	global distanceMatrix

	idx = json.loads(request.args.get('idx'))

	# pdb.set_trace()

	[mappedX_sub, cl_idx_sub, Wtopk_idx_sub] = eng.sub_topic(idx,nargout=3)

	Wtopk_sub = []
	for idxArray in Wtopk_idx_sub:
		tempArray = []
		for topicIdx in idxArray:
			tempArray.append(voca[int(topicIdx)-1])
		Wtopk_sub.append(tempArray)

	cl_idx_sub = cl_idx_sub[0]
	cl_idx_sub = np.array(cl_idx_sub).tolist()

	print distanceMatrix.shape

	distanceMatrix_sub = distanceMatrix[idx,:][:,idx]

	sameTopicWeight = 1
	differentTopicWeight = 1
	distanceMatrix_sub = supervisedTSNE(distanceMatrix_sub, cl_idx_sub,
		sameTopicWeight=sameTopicWeight, differentTopicWeight=differentTopicWeight)
	distanceMatrix_sub = distanceMatrix_sub.tolist()

	return json.dumps({'distanceMatrix':distanceMatrix_sub, 'cl_idx_sub':cl_idx_sub, 'Wtopk_sub':Wtopk_sub})

@app.route('/get_subTopic_tsne')
def get_subTopic_tsne():
	global eng
	global voca
	idx = json.loads(request.args.get('idx'))

	[mappedX_sub, cl_idx_sub, Wtopk_idx_sub] = eng.sub_topic_tsne(idx,nargout=3)

	print mappedX_sub

	Wtopk_sub = []
	for idxArray in Wtopk_idx_sub:
		tempArray = []
		for idx in idxArray:
			tempArray.append(voca[int(idx)-1])
		Wtopk_sub.append(tempArray)

	cl_idx_sub = cl_idx_sub[0]

	mappedX_sub = np.array(mappedX_sub).tolist()
	cl_idx_sub = np.array(cl_idx_sub).tolist()

	return json.dumps({'mappedX_sub':mappedX_sub, 'cl_idx_sub':cl_idx_sub, 'Wtopk_sub':Wtopk_sub})

# keyword 입력받음
@app.route('/')
def form():
	global cl_idx
	global Wtopk
	global distanceMatrix_main

	return render_template('tsne.html', cl_idx=cl_idx, Wtopk= Wtopk, distanceMatrix=distanceMatrix_main)

# Execute the main program
if __name__ == '__main__':
	before__first_request_()
	app.run(host='0.0.0.0',port=5004)
<commit_msg>error fix(change index to matlab index starting 1)<commit_after># -*- coding: utf-8 -*-

from werkzeug.contrib.profiler import ProfilerMiddleware
from flask import Flask, request, g, render_template
from flask.ext.triangle import Triangle
from scipy import sparse, io
from sklearn.metrics.pairwise import pairwise_distances
import numpy as np
from matlab import engine
import os, json, time
from flask.ext.cors import CORS
import pdb
from jinja2 import Environment

# Configuration
app = Flask(__name__, static_path='/static')
Triangle(app)
CORS(app)
app.config['CORS_HEADERS'] = 'Content-Type'

app.config['DEBUG'] = True
app.config.from_object(__name__)
app.config.from_envvar('FLASKR_SETTINGS', silent=True)

app.config['PROFILE'] = True
app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[10])
# @app.after_request
# def after_request(response):
#   response.headers.add('Access-Control-Allow-Origin', '*')
#   response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
#   response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE')
#   return response

def supervisedTSNE(distanceMatrix, cl_idx, sameTopicWeight=0.9, differentTopicWeight=1.1):
	n = distanceMatrix.shape[0]
	for i in xrange(n):
		i_topic = cl_idx[i]
		for j in xrange(n):
			if i==j:
				continue
			elif i_topic==cl_idx[j]:
				distanceMatrix[i,j]*=sameTopicWeight
			else:
				distanceMatrix[i,j]*=differentTopicWeight
		distanceMatrix[i,:] /= distanceMatrix[i,:].max()

	return np.round(distanceMatrix,decimals=4)

# Routing
# @app.before_first_request
def before__first_request_():
	global eng
	global mappedX
	global cl_idx
	global Wtopk
	global voca
	global distanceMatrix
	global distanceMatrix_main

	tic = time.time()
	print 'Starting matlab - ',
	eng = engine.start_matlab()
	eng.cd(os.path.dirname(os.getcwd()))
	print "%.4f" % (time.time()-tic)

	#tic = time.time()
	##print "Get data - ",
	#[mappedX, cl_idx, Wtopk_idx,voca] = eng.main_topic(nargout=4)
	#print "%.4f" % (time.time()-tic)

	tic = time.time()
	eng.main_topic2(nargout=0)

	mappedX_ = eng.workspace['mappedX']
	cl_idx_ = eng.workspace['cl_idx']
	Wtopk_idx_ = eng.workspace['Wtopk_idx']
	voca_ = eng.workspace['dict']
	print "%.4f" % (time.time()-tic)

	distanceMatrix = io.loadmat('./../tdm2.mat')['DD']

	tic = time.time()
	print "Calculate data - ",

	Wtopk = []
	for idxArray in Wtopk_idx:
		tempArray = []
		for idx in idxArray:
			tempArray.append(voca[int(idx)-1])
		Wtopk.append(tempArray)

	cl_idx = cl_idx[0]

	cl_idx = cl_idx
	distanceMatrix = distanceMatrix

	cl_idx = cl_idx
	distanceMatrix = distanceMatrix

	sameTopicWeight = 0.9
	differentTopicWeight = 1.1
	distanceMatrix_main = supervisedTSNE(distanceMatrix, cl_idx,
		sameTopicWeight=sameTopicWeight, differentTopicWeight=differentTopicWeight)

	print "%.4f" % (time.time()-tic)

	distanceMatrix_main = distanceMatrix_main.tolist()

@app.teardown_request
def teardown_request(exception):
	print('Teardown arose!'.format(exception))

@app.route('/get_subTopic')
def get_subTopic():
	global eng
	global voca
	global distanceMatrix

	idx = json.loads(request.args.get('idx'))
	idx = [i+1 for i in idx]

	# pdb.set_trace()

	[mappedX_sub, cl_idx_sub, Wtopk_idx_sub] = eng.sub_topic(idx,nargout=3)

	Wtopk_sub = []
	for idxArray in Wtopk_idx_sub:
		tempArray = []
		for topicIdx in idxArray:
			tempArray.append(voca[int(topicIdx)-1])
		Wtopk_sub.append(tempArray)

	cl_idx_sub = cl_idx_sub[0]
	cl_idx_sub = np.array(cl_idx_sub).tolist()

	print distanceMatrix.shape

	distanceMatrix_sub = distanceMatrix[idx,:][:,idx]

	sameTopicWeight = 1
	differentTopicWeight = 1
	distanceMatrix_sub = supervisedTSNE(distanceMatrix_sub, cl_idx_sub,
		sameTopicWeight=sameTopicWeight, differentTopicWeight=differentTopicWeight)
	distanceMatrix_sub = distanceMatrix_sub.tolist()

	return json.dumps({'distanceMatrix':distanceMatrix_sub, 'cl_idx_sub':cl_idx_sub, 'Wtopk_sub':Wtopk_sub})

@app.route('/get_subTopic_tsne')
def get_subTopic_tsne():
	global eng
	global voca
	idx = json.loads(request.args.get('idx'))

	[mappedX_sub, cl_idx_sub, Wtopk_idx_sub] = eng.sub_topic_tsne(idx,nargout=3)

	print mappedX_sub

	Wtopk_sub = []
	for idxArray in Wtopk_idx_sub:
		tempArray = []
		for idx in idxArray:
			tempArray.append(voca[int(idx)-1])
		Wtopk_sub.append(tempArray)

	cl_idx_sub = cl_idx_sub[0]

	mappedX_sub = np.array(mappedX_sub).tolist()
	cl_idx_sub = np.array(cl_idx_sub).tolist()

	return json.dumps({'mappedX_sub':mappedX_sub, 'cl_idx_sub':cl_idx_sub, 'Wtopk_sub':Wtopk_sub})

# keyword 입력받음
@app.route('/')
def form():
	global cl_idx
	global Wtopk
	global distanceMatrix_main

	return render_template('tsne.html', cl_idx=cl_idx, Wtopk= Wtopk, distanceMatrix=distanceMatrix_main)

# Execute the main program
if __name__ == '__main__':
	before__first_request_()
	app.run(host='0.0.0.0',port=5004)
<|endoftext|>


<commit_before>'''
See LICENSE for license information.

This file contains the server's loop control. It is decoupled from
the code run within the main loop. This allows the main loop to be reloaded
and updated at runtime.
'''

##
# Built-in modules
##

import sys

##
# External modules
##

try:
    from twisted.internet import service
    from twisted.application.internet import TCPServer
except ImportError:
    sys.err.println("The Twisted module must be installed and accessible.")
    sys.exit(1)

##
# Built-in modules
##

from serverfactory import DogiAdmin
from config import Configurator

##
# Setup the service for the reactor
##

config = Configurator()
das = DogiServer(config)

application = service.Application('dogiadminserver')
TCPServer(3644, das.getProtocolFactory()).setServiceParent(
  service.IServiceCollection(application))

##
# TODO
# start engine if run as a script
##

#if __name__ == '__main__':
#    reactor.run()
<commit_msg>minor typo fix<commit_after>'''
See LICENSE for license information.

This file contains the server's loop control. It is decoupled from
the code run within the main loop. This allows the main loop to be reloaded
and updated at runtime.
'''

##
# Built-in modules
##

import sys

##
# External modules
##

try:
    from twisted.internet import service
    from twisted.application.internet import TCPServer
except ImportError:
    sys.err.println("The Twisted module must be installed and accessible.")
    sys.exit(1)

##
# Internal modules
##

from serverfactory import DogiAdmin
from config import Configurator

##
# Setup the service for the reactor
##

config = Configurator()
das = DogiServer(config)

application = service.Application('dogiadminserver')
TCPServer(3644, das.getProtocolFactory()).setServiceParent(
  service.IServiceCollection(application))

##
# TODO
# start engine if run as a script
##

#if __name__ == '__main__':
#    reactor.run()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
import os
import time
import struct
import marshal
import traceback

def pack_int(i):
	return struct.pack(">i", i)

def pack_short(i):
	return struct.pack(">h", i)

def pack_byte(i):
	return struct.pack(">b", i)

def pack_long(i):
	return struct.pack(">q", i)

def pack_unsigned_int(i):
	return struct.pack(">I", i)

def pack_unsigned_short(i):
	return struct.pack(">H", i)

def pack_unsigned_byte(i):
	return struct.pack(">B", i)

def pack_unsigned_long(i):
	return struct.pack(">Q", i)

def unpack_int(s):
	return struct.unpack(">i", s)[0]

def unpack_short(s):
	return struct.unpack(">h", s)[0]

def unpack_byte(s):
	return struct.unpack(">b", s)[0]

def unpack_long(i):
	return struct.unpack(">q", i)[0]

def unpack_unsigned_int(s):
	return struct.unpack(">I", s)[0]

def unpack_unsigned_short(s):
	return struct.unpack(">H", s)[0]

def unpack_unsigned_byte(s):
	return struct.unpack(">B", s)[0]

def unpack_unsigned_long(s):
	return struct.unpack(">Q", s)[0]

def pack_str(string):
	#65636f -> 04 65636f00
	if not string:
		return "\x01\x00"
	string += "\x00"
	return struct.pack(">B", len(string))+string #unsigned byte + char*

def unpack_str(code):
	string, length = unpack_raw(code)
	while string.endswith("\x00"):
		string = string[:-1]
	return string, length

def unpack_raw(code):
	length_data = code[:1]
	if length_data == "": return ""
	length = struct.unpack(">B", length_data)[0] #unsigned byte
	string = code[1:length+1]
	return string, length+1

def io_unpack_int(io):
	return struct.unpack(">i", io.read(4))[0]

def io_unpack_short(io):
	return struct.unpack(">h", io.read(2))[0]

def io_unpack_byte(io):
	return struct.unpack(">b", io.read(1))[0]

def io_unpack_long(io):
	return struct.unpack(">q", io.read(8))[0]

def io_unpack_unsigned_int(io):
	return struct.unpack(">I", io.read(4))[0]

def io_unpack_unsigned_short(io):
	return struct.unpack(">H", io.read(2))[0]

def io_unpack_unsigned_byte(io):
	return struct.unpack(">B", io.read(1))[0]

def io_unpack_unsigned_long(io):
	return struct.unpack(">Q", io.read(8))[0]

def io_unpack_str(io):
	string = io_unpack_raw(io)
	while string.endswith("\x00"):
		string = string[:-1]
	return string

def io_unpack_short_str(io):
	string = io_unpack_short_raw(io)
	while string.endswith("\x00"):
		string = string[:-1]
	return string

def io_unpack_raw(io):
	length_data = io.read(1)
	if length_data == "": return ""
	length = struct.unpack(">B", length_data)[0] #unsigned byte
	string = io.read(length)
	return string

def io_unpack_short_raw(io):
	length_data = io.read(2)
	if length_data == "": return ""
	length = struct.unpack(">H", length_data)[0] #unsigned short
	string = io.read(length)
	return string

def pack_array(mod, array):
	if type(array) not in (list, tuple):
		array = tuple(array)
	length = len(array)
	if length >= 253:
		string = pack_unsigned_byte(253)+pack_unsigned_int(length)
	else:
		string = pack_unsigned_byte(length)
	for i in array:
		string += mod(i)
	return string

def io_unpack_array(mod, io):
	array = []
	length = io_unpack_unsigned_byte(io)
	if length == 253:
		length = io_unpack_unsigned_int(io)
	for i in xrange(length):
		array.append(mod(io))
	return array

def pack_user_data(pack, user, attr):
	result = "\x04"
	for p in user.pc_list:
		result += pack(getattr(p, attr) if p else 0)
	return result

def pack_user_byte(*args):
	return pack_user_data(pack_byte, *args)

def pack_user_short(*args):
	return pack_user_data(pack_short, *args)

def pack_user_int(*args):
	return pack_user_data(pack_int, *args)

def pack_pict_id(item, item_type):
	return pack_int(item.get_pict_id(item_type) if item else 0)

def pack_item_int_attr(item, item_type, attr):
	return pack_int(item.get_int_attr(attr, item_type) if item else 0)

def pack_item_short_attr(item, attr, item_type):
	return pack_short(item.get_int_attr(attr, item_type) if item else 0)

def pack_item_byte_attr(item, attr, item_type):
	return pack_byte(item.get_int_attr(attr, item_type) if item else 0)

def pack_item_unsigned_byte_attr(item, attr, item_type):
	return pack_unsigned_byte(item.get_int_attr(attr, item_type) if item else 0)

def pack_item_str_attr(item, attr, item_type):
	return pack_str(item.get_str_attr(attr, item_type) if item else 0)
<commit_msg>small fixes in packet_struct<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
import os
import time
import struct
import marshal
import traceback
try: from cStringIO import StringIO
except: from StringIO import StringIO

def pack_int(i):
	return struct.pack(">i", i)

def pack_short(i):
	return struct.pack(">h", i)

def pack_byte(i):
	return struct.pack(">b", i)

def pack_long(i):
	return struct.pack(">q", i)

def pack_unsigned_int(i):
	return struct.pack(">I", i)

def pack_unsigned_short(i):
	return struct.pack(">H", i)

def pack_unsigned_byte(i):
	return struct.pack(">B", i)

def pack_unsigned_long(i):
	return struct.pack(">Q", i)

def unpack_int(s):
	return struct.unpack(">i", s)[0]

def unpack_short(s):
	return struct.unpack(">h", s)[0]

def unpack_byte(s):
	return struct.unpack(">b", s)[0]

def unpack_long(i):
	return struct.unpack(">q", i)[0]

def unpack_unsigned_int(s):
	return struct.unpack(">I", s)[0]

def unpack_unsigned_short(s):
	return struct.unpack(">H", s)[0]

def unpack_unsigned_byte(s):
	return struct.unpack(">B", s)[0]

def unpack_unsigned_long(s):
	return struct.unpack(">Q", s)[0]

def unpack_str(code):
	io = StringIO(code)
	string = io_unpack_str(io)
	return string, io.tell()

def unpack_raw(code):
	io = StringIO(code)
	string = io_unpack_raw(io)
	return string, io.tell()

def unpack_array(code):
	io = StringIO(code)
	array = io_unpack_array(StringIO(code))
	return array, io.tell()

def io_unpack_int(io):
	return struct.unpack(">i", io.read(4))[0]

def io_unpack_short(io):
	return struct.unpack(">h", io.read(2))[0]

def io_unpack_byte(io):
	return struct.unpack(">b", io.read(1))[0]

def io_unpack_long(io):
	return struct.unpack(">q", io.read(8))[0]

def io_unpack_unsigned_int(io):
	return struct.unpack(">I", io.read(4))[0]

def io_unpack_unsigned_short(io):
	return struct.unpack(">H", io.read(2))[0]

def io_unpack_unsigned_byte(io):
	return struct.unpack(">B", io.read(1))[0]

def io_unpack_unsigned_long(io):
	return struct.unpack(">Q", io.read(8))[0]

def pack_str(string):
	#65636f -> 04 65636f00
	if not string:
		return "\x01\x00"
	string += "\x00"
	length = len(string)
	if length >= 253:
		return pack_unsigned_byte(253)+pack_unsigned_int(length)+string
	else:
		return pack_unsigned_byte(length)+string

def io_unpack_str(io):
	string = io_unpack_raw(io)
	while string.endswith("\x00"):
		string = string[:-1]
	return string

def io_unpack_raw(io):
	length = io_unpack_unsigned_byte(io)
	if length >= 253:
		length = io_unpack_unsigned_int(io)
	return io.read(length)

def io_unpack_short_str(io):
	string = io_unpack_short_raw(io)
	while string.endswith("\x00"):
		string = string[:-1]
	return string

def io_unpack_short_raw(io):
	length_data = io.read(2)
	if length_data == "":
		return ""
	length = unpack_unsigned_short(length_data)
	return io.read(length)

def pack_array(mod, array):
	if type(array) not in (list, tuple):
		array = tuple(array)
	length = len(array)
	if length >= 253:
		string = pack_unsigned_byte(253)+pack_unsigned_int(length)
	else:
		string = pack_unsigned_byte(length)
	for i in array:
		string += mod(i)
	return string

def io_unpack_array(mod, io):
	array = []
	length = io_unpack_unsigned_byte(io)
	if length >= 253:
		length = io_unpack_unsigned_int(io)
	for i in xrange(length):
		array.append(mod(io))
	return array

def pack_user_data(pack, user, attr):
	result = "\x04"
	for p in user.pc_list:
		result += pack(getattr(p, attr) if p else 0)
	return result

def pack_user_byte(*args):
	return pack_user_data(pack_byte, *args)

def pack_user_short(*args):
	return pack_user_data(pack_short, *args)

def pack_user_int(*args):
	return pack_user_data(pack_int, *args)

def pack_pict_id(item, item_type):
	return pack_int(item.get_pict_id(item_type) if item else 0)

def pack_item_int_attr(item, item_type, attr):
	return pack_int(item.get_int_attr(attr, item_type) if item else 0)

def pack_item_short_attr(item, attr, item_type):
	return pack_short(item.get_int_attr(attr, item_type) if item else 0)

def pack_item_byte_attr(item, attr, item_type):
	return pack_byte(item.get_int_attr(attr, item_type) if item else 0)

def pack_item_unsigned_byte_attr(item, attr, item_type):
	return pack_unsigned_byte(item.get_int_attr(attr, item_type) if item else 0)

def pack_item_str_attr(item, attr, item_type):
	return pack_str(item.get_str_attr(attr, item_type) if item else 0)
<|endoftext|>


<commit_before>from bs4 import BeautifulSoup

def parse_links_for_text(string_with_links):
    strings = [link.get_text()
               for link in BeautifulSoup(string_with_links, 'html.parser').find_all('a')]
    splitted = [' '.join(s.split()).strip() for s in strings]
    return [s for s in splitted if s]
<commit_msg>use regexes instead of BeautifulSoup in parse_links_for_text<commit_after>import re
linkre = re.compile(r'<a.*?>(.*?)<\/a>')

def parse_links_for_text(string_with_links):
    matches = linkre.finditer(string_with_links)
    strings = [link.group(1) for link in matches if link]
    splitted = [' '.join(s.split()).strip() for s in strings]
    return [s for s in splitted if s]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import subprocess

from setuptools import find_packages, setup

src_dir = os.path.dirname(__file__)
home_dir = os.path.abspath(os.path.join(src_dir, "../../.."))

install_requires = [
    # DAX/Workflow
    "PyYAML",
    # Python 2 compatibility
    "six>=1.9.0",
    # pegasus-init
    "Jinja2==2.10.3",
    "Flask-SQLAlchemy==2.3.2",
    "boto==2.48.0",
    "pamela==1.0.0",
    "globus-sdk==1.4.1",
    "pika==1.1.0",
    # Python 2.6
    'Flask==0.12.4;python_version<="2.6"',
    'Flask-Cache==0.13.1;python_version<="2.6"',
    'requests==2.18.4;python_version<="2.6"',
    'ordereddict==1.1;python_version<="2.6"',
    'argparse==1.4.0;python_version<="2.6"',
    'sqlalchemy==1.1.15;python_version<="2.6"',
    # Python 2.7+
    'Flask==1.0.2;python_version>"2.6"',
    'Flask-Caching;python_version>"2.6"',
    'requests==2.21.0;python_version>"2.6"',
    'sqlalchemy==1.2.1;python_version>"2.6"',
    # Python 3 Backport
    'pathlib2;python_version<"3.0"',
    'functools32;python_version<"3.0"',
    # 'dataclasses;python_version=="3.6"',
]

excludes = ["Pegasus.test*"]

#
# Install conditional dependencies
#
def setup_installer_dependencies():
    global install_requires

    if subprocess.call(["which", "mysql_config"]) == 0:
        install_requires.append('MySQL-Python;python_version<="2.6"')
        install_requires.append('mysqlclient;python_version>"2.6"')

#
# Utility function to read the pegasus Version.in file
#
def read_version():
    return (
        subprocess.Popen(
            "%s/release-tools/getversion" % home_dir, stdout=subprocess.PIPE, shell=True
        )
        .communicate()[0]
        .decode()
        .strip()
    )

#
# Utility function to read the README file.
#
def read(fname):
    return open(os.path.join(src_dir, fname)).read()

def find_package_data(dirname):
    def find_paths(dirname):
        items = []
        for fname in os.listdir(dirname):
            path = os.path.join(dirname, fname)
            if os.path.isdir(path):
                items += find_paths(path)
            elif not path.endswith(".py") and not path.endswith(".pyc"):
                items.append(path)
        return items

    items = find_paths(dirname)
    return [path.replace(dirname, "") for path in items]

setup_installer_dependencies()

setup(
    name="pegasus-wms",
    version=read_version(),
    author="Pegasus Team",
    author_email="pegasus@isi.edu",
    description="Pegasus Workflow Management System Python API",
    long_description=read("README"),
    license="Apache2",
    url="http://pegasus.isi.edu",
    python_requires=">=2.6,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*",
    keywords=["scientific workflows"],
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Intended Audience :: Developers",
        "Intended Audience :: Science/Research",
        "Operating System :: Unix",
        "Programming Language :: Python",
        "Programming Language :: Python :: 2",
        "Programming Language :: Python :: 2.6",
        "Programming Language :: Python :: 2.7",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.5",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
        "Topic :: Scientific/Engineering",
        "Topic :: Utilities",
        "License :: OSI Approved :: Apache Software License",
    ],
    packages=find_packages(exclude=excludes),
    package_data={"Pegasus.service": find_package_data("Pegasus/service/")},
    include_package_data=True,
    zip_safe=False,
    install_requires=install_requires,
    extras_require={"postgresql": ["psycopg2"], "mysql": []},
)
<commit_msg>PM-1235 Update python_requires to exclude Py 3.4<commit_after># -*- coding: utf-8 -*-

import os
import subprocess

from setuptools import find_packages, setup

src_dir = os.path.dirname(__file__)
home_dir = os.path.abspath(os.path.join(src_dir, "../../.."))

install_requires = [
    # DAX/Workflow
    "PyYAML",
    # Python 2 compatibility
    "six>=1.9.0",
    # pegasus-init
    "Jinja2==2.10.3",
    "Flask-SQLAlchemy==2.3.2",
    "boto==2.48.0",
    "pamela==1.0.0",
    "globus-sdk==1.4.1",
    "pika==1.1.0",
    # Python 2.6
    'Flask==0.12.4;python_version<="2.6"',
    'Flask-Cache==0.13.1;python_version<="2.6"',
    'requests==2.18.4;python_version<="2.6"',
    'ordereddict==1.1;python_version<="2.6"',
    'argparse==1.4.0;python_version<="2.6"',
    'sqlalchemy==1.1.15;python_version<="2.6"',
    # Python 2.7+
    'Flask==1.0.2;python_version>"2.6"',
    'Flask-Caching;python_version>"2.6"',
    'requests==2.21.0;python_version>"2.6"',
    'sqlalchemy==1.2.1;python_version>"2.6"',
    # Python 3 Backport
    'pathlib2;python_version<"3.0"',
    'functools32;python_version<"3.0"',
    # 'dataclasses;python_version=="3.6"',
]

excludes = ["Pegasus.test*"]

#
# Install conditional dependencies
#
def setup_installer_dependencies():
    global install_requires

    if subprocess.call(["which", "mysql_config"]) == 0:
        install_requires.append('MySQL-Python;python_version<="2.6"')
        install_requires.append('mysqlclient;python_version>"2.6"')

#
# Utility function to read the pegasus Version.in file
#
def read_version():
    return (
        subprocess.Popen(
            "%s/release-tools/getversion" % home_dir, stdout=subprocess.PIPE, shell=True
        )
        .communicate()[0]
        .decode()
        .strip()
    )

#
# Utility function to read the README file.
#
def read(fname):
    return open(os.path.join(src_dir, fname)).read()

def find_package_data(dirname):
    def find_paths(dirname):
        items = []
        for fname in os.listdir(dirname):
            path = os.path.join(dirname, fname)
            if os.path.isdir(path):
                items += find_paths(path)
            elif not path.endswith(".py") and not path.endswith(".pyc"):
                items.append(path)
        return items

    items = find_paths(dirname)
    return [path.replace(dirname, "") for path in items]

setup_installer_dependencies()

setup(
    name="pegasus-wms",
    version=read_version(),
    author="Pegasus Team",
    author_email="pegasus@isi.edu",
    description="Pegasus Workflow Management System Python API",
    long_description=read("README"),
    license="Apache2",
    url="http://pegasus.isi.edu",
    python_requires=">=2.6,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*",
    keywords=["scientific workflows"],
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Intended Audience :: Developers",
        "Intended Audience :: Science/Research",
        "Operating System :: Unix",
        "Programming Language :: Python",
        "Programming Language :: Python :: 2",
        "Programming Language :: Python :: 2.6",
        "Programming Language :: Python :: 2.7",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.5",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
        "Topic :: Scientific/Engineering",
        "Topic :: Utilities",
        "License :: OSI Approved :: Apache Software License",
    ],
    packages=find_packages(exclude=excludes),
    package_data={"Pegasus.service": find_package_data("Pegasus/service/")},
    include_package_data=True,
    zip_safe=False,
    install_requires=install_requires,
    extras_require={"postgresql": ["psycopg2"], "mysql": []},
)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
$Id$

Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/)
See the file 'doc/COPYING' for copying permission
"""

import threading
import time

from lib.core.common import clearConsoleLine
from lib.core.common import dataToSessionFile
from lib.core.common import dataToStdout
from lib.core.common import filterListValue
from lib.core.common import getFileItems
from lib.core.common import Backend
from lib.core.common import getPageTextWordsSet
from lib.core.common import popValue
from lib.core.common import pushValue
from lib.core.common import randomInt
from lib.core.common import readInput
from lib.core.common import safeStringFormat
from lib.core.common import safeSQLIdentificatorNaming
from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger
from lib.core.enums import DBMS
from lib.core.exception import sqlmapMissingMandatoryOptionException
from lib.core.exception import sqlmapThreadException
from lib.core.settings import METADB_SUFFIX
from lib.core.session import safeFormatString
from lib.request import inject

def tableExists(tableFile, regex=None):
    tables = getFileItems(tableFile, lowercase=Backend.getIdentifiedDbms() in (DBMS.ACCESS), unique=True)
    retVal = []

    infoMsg = "checking table existence using items from '%s'" % tableFile
    logger.info(infoMsg)

    infoMsg = "adding words used on web page to the check list"
    logger.info(infoMsg)
    pageWords = getPageTextWordsSet(kb.originalPage)

    for word in pageWords:
        word = word.lower()

        if len(word) > 2 and not word[0].isdigit() and word not in tables:
            tables.append(word)

    tables = filterListValue(tables, regex)
    count = [0]
    length = len(tables)
    threads = []
    tbllock = threading.Lock()
    iolock = threading.Lock()
    kb.threadContinue = True
    kb.bruteMode = True

    def tableExistsThread():
        while count[0] < length and kb.threadContinue:
            tbllock.acquire()
            table = safeSQLIdentificatorNaming(tables[count[0]])
            count[0] += 1
            tbllock.release()

            if conf.db and not conf.db.endswith(METADB_SUFFIX):
                fullTableName = "%s%s%s" % (conf.db, '..' if Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE) else '.', table)
            else:
                fullTableName = table

            result = inject.checkBooleanExpression("%s" % safeStringFormat("EXISTS(SELECT %d FROM %s)", (randomInt(1), fullTableName)))

            iolock.acquire()

            if result:
                retVal.append(table)

                dataToSessionFile("[%s][%s][%s][TABLE_EXISTS][%s]\n" % (conf.url,\
                  kb.injection.place, safeFormatString(conf.parameters[kb.injection.place]),\
                  safeFormatString(fullTableName)))

                if conf.verbose in (1, 2):
                    clearConsoleLine(True)
                    infoMsg = "\r[%s] [INFO] retrieved: %s\n" % (time.strftime("%X"), table)
                    dataToStdout(infoMsg, True)

            if conf.verbose in (1, 2):
                status = '%d/%d items (%d%s)' % (count[0], length, round(100.0*count[0]/length), '%')
                dataToStdout("\r[%s] [INFO] tried %s" % (time.strftime("%X"), status), True)

            iolock.release()

    if conf.threads > 1:
        infoMsg = "starting %d threads" % conf.threads
        logger.info(infoMsg)
    else:
        message = "please enter number of threads? [Enter for %d (current)] " % conf.threads
        choice = readInput(message, default=str(conf.threads))
        if choice and choice.isdigit():
            conf.threads = int(choice)

    if conf.threads == 1:
        warnMsg = "running in a single-thread mode. This could take a while."
        logger.warn(warnMsg)

    # Start the threads
    for numThread in range(conf.threads):
        thread = threading.Thread(target=tableExistsThread, name=str(numThread))
        thread.start()
        threads.append(thread)

    # And wait for them to all finish
    try:
        alive = True

        while alive:
            alive = False

            for thread in threads:
                if thread.isAlive():
                    alive = True
                    thread.join(5)
    except KeyboardInterrupt:
        kb.threadContinue = False
        kb.threadException = True

        print
        logger.debug("waiting for threads to finish")

        warnMsg = "user aborted during common table existence check. "
        warnMsg += "sqlmap will display some tables only"
        logger.warn(warnMsg)

        try:
            while (threading.activeCount() > 1):
                pass

        except KeyboardInterrupt:
            raise sqlmapThreadException, "user aborted"
    finally:
        kb.bruteMode = False
        kb.threadContinue = True
        kb.threadException = False

    clearConsoleLine(True)
    dataToStdout("\n")

    if not retVal:
        warnMsg = "no table found"
        logger.warn(warnMsg)
    else:
        items = set()
        for item in retVal:
            if item.lower() in items:
                continue
            else:
                items.add(item.lower())
            if not kb.data.cachedTables.has_key(conf.db):
                kb.data.cachedTables[conf.db] = [item]
            else:
                kb.data.cachedTables[conf.db].append(item)

    return kb.data.cachedTables

def columnExists(columnFile, regex=None):
    if not conf.tbl:
        errMsg = "missing table parameter"
        raise sqlmapMissingMandatoryOptionException, errMsg

    columns = getFileItems(columnFile, unique=True)
    columns = filterListValue(columns, regex)

    if conf.db and not conf.db.endswith(METADB_SUFFIX):
        table = "%s%s%s" % (conf.db, '..' if Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE) else '.', conf.tbl)
    else:
        table = conf.tbl
    table = safeSQLIdentificatorNaming(table)

    retVal = []
    infoMsg = "checking column existence using items from '%s'" % columnFile
    logger.info(infoMsg)

    count = [0]
    length = len(columns)
    threads = []
    collock = threading.Lock()
    iolock = threading.Lock()
    kb.threadContinue = True
    kb.bruteMode = True

    def columnExistsThread():
        while count[0] < length and kb.threadContinue:
            collock.acquire()
            column = safeSQLIdentificatorNaming(columns[count[0]])
            count[0] += 1
            collock.release()

            result = inject.checkBooleanExpression("%s" % safeStringFormat("EXISTS(SELECT %s FROM %s)", (column, table)))

            iolock.acquire()

            if result:
                retVal.append(column)

                if conf.verbose in (1, 2):
                    clearConsoleLine(True)
                    infoMsg = "\r[%s] [INFO] retrieved: %s\n" % (time.strftime("%X"), column)
                    dataToStdout(infoMsg, True)

            if conf.verbose in (1, 2):
                status = '%d/%d items (%d%s)' % (count[0], length, round(100.0*count[0]/length), '%')
                dataToStdout("\r[%s] [INFO] tried %s" % (time.strftime("%X"), status), True)

            iolock.release()

    if conf.threads > 1:
        infoMsg = "starting %d threads" % conf.threads
        logger.info(infoMsg)
    else:
        message = "please enter number of threads? [Enter for %d (current)] " % conf.threads
        choice = readInput(message, default=str(conf.threads))
        if choice and choice.isdigit():
            conf.threads = int(choice)

    if conf.threads == 1:
        warnMsg = "running in a single-thread mode. This could take a while."
        logger.warn(warnMsg)

    # Start the threads
    for numThread in range(conf.threads):
        thread = threading.Thread(target=columnExistsThread, name=str(numThread))
        thread.start()
        threads.append(thread)

    # And wait for them to all finish
    try:
        alive = True

        while alive:
            alive = False

            for thread in threads:
                if thread.isAlive():
                    alive = True
                    thread.join(5)
    except KeyboardInterrupt:
        kb.threadContinue = False
        kb.threadException = True

        print
        logger.debug("waiting for threads to finish")

        warnMsg = "user aborted during common column existence check. "
        warnMsg += "sqlmap will display some columns only"
        logger.warn(warnMsg)

        try:
            while (threading.activeCount() > 1):
                pass

        except KeyboardInterrupt:
            raise sqlmapThreadException, "user aborted"
    finally:
        kb.bruteMode = False
        kb.threadContinue = True
        kb.threadException = False

    clearConsoleLine(True)
    dataToStdout("\n")

    if not retVal:
        warnMsg = "no column found"
        logger.warn(warnMsg)
    else:
        columns = {}

        for column in retVal:
            result = inject.checkBooleanExpression("%s" % safeStringFormat("EXISTS(SELECT %s FROM %s WHERE ROUND(%s)=ROUND(%s))", (column, table, column, column)))

            if result:
                columns[column] = 'numeric'
            else:
                columns[column] = 'non-numeric'

            dataToSessionFile("[%s][%s][%s][COLUMN_EXISTS][%s|%s %s]\n" % (conf.url, kb.injection.place,\
              safeFormatString(conf.parameters[kb.injection.place]), safeFormatString(table),\
              safeFormatString(column), safeFormatString(columns[column])))

        kb.data.cachedColumns[conf.db] = {conf.tbl: columns}

    return kb.data.cachedColumns
<commit_msg>better one for previous commit<commit_after>#!/usr/bin/env python

"""
$Id$

Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/)
See the file 'doc/COPYING' for copying permission
"""

import threading
import time

from lib.core.common import clearConsoleLine
from lib.core.common import dataToSessionFile
from lib.core.common import dataToStdout
from lib.core.common import filterListValue
from lib.core.common import getFileItems
from lib.core.common import Backend
from lib.core.common import getPageTextWordsSet
from lib.core.common import popValue
from lib.core.common import pushValue
from lib.core.common import randomInt
from lib.core.common import readInput
from lib.core.common import safeStringFormat
from lib.core.common import safeSQLIdentificatorNaming
from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger
from lib.core.enums import DBMS
from lib.core.exception import sqlmapMissingMandatoryOptionException
from lib.core.exception import sqlmapThreadException
from lib.core.settings import METADB_SUFFIX
from lib.core.session import safeFormatString
from lib.request import inject

def tableExists(tableFile, regex=None):
    tables = getFileItems(tableFile, lowercase=Backend.getIdentifiedDbms() in (DBMS.ACCESS), unique=True)
    retVal = []

    infoMsg = "checking table existence using items from '%s'" % tableFile
    logger.info(infoMsg)

    infoMsg = "adding words used on web page to the check list"
    logger.info(infoMsg)
    pageWords = getPageTextWordsSet(kb.originalPage)

    for word in pageWords:
        word = word.lower()

        if len(word) > 2 and not word[0].isdigit() and word not in tables:
            tables.append(word)

    tables = filterListValue(tables, regex)
    count = [0]
    length = len(tables)
    threads = []
    items = set()
    tbllock = threading.Lock()
    iolock = threading.Lock()
    kb.threadContinue = True
    kb.bruteMode = True

    def tableExistsThread():
        while count[0] < length and kb.threadContinue:
            tbllock.acquire()
            table = safeSQLIdentificatorNaming(tables[count[0]])
            count[0] += 1
            tbllock.release()

            if conf.db and not conf.db.endswith(METADB_SUFFIX):
                fullTableName = "%s%s%s" % (conf.db, '..' if Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE) else '.', table)
            else:
                fullTableName = table

            result = inject.checkBooleanExpression("%s" % safeStringFormat("EXISTS(SELECT %d FROM %s)", (randomInt(1), fullTableName)))

            iolock.acquire()

            if result and table.lower() not in items:
                retVal.append(table)

                items.add(table.lower())

                dataToSessionFile("[%s][%s][%s][TABLE_EXISTS][%s]\n" % (conf.url,\
                  kb.injection.place, safeFormatString(conf.parameters[kb.injection.place]),\
                  safeFormatString(fullTableName)))

                if conf.verbose in (1, 2):
                    clearConsoleLine(True)
                    infoMsg = "\r[%s] [INFO] retrieved: %s\n" % (time.strftime("%X"), table)
                    dataToStdout(infoMsg, True)

            if conf.verbose in (1, 2):
                status = '%d/%d items (%d%s)' % (count[0], length, round(100.0*count[0]/length), '%')
                dataToStdout("\r[%s] [INFO] tried %s" % (time.strftime("%X"), status), True)

            iolock.release()

    if conf.threads > 1:
        infoMsg = "starting %d threads" % conf.threads
        logger.info(infoMsg)
    else:
        message = "please enter number of threads? [Enter for %d (current)] " % conf.threads
        choice = readInput(message, default=str(conf.threads))
        if choice and choice.isdigit():
            conf.threads = int(choice)

    if conf.threads == 1:
        warnMsg = "running in a single-thread mode. This could take a while."
        logger.warn(warnMsg)

    # Start the threads
    for numThread in range(conf.threads):
        thread = threading.Thread(target=tableExistsThread, name=str(numThread))
        thread.start()
        threads.append(thread)

    # And wait for them to all finish
    try:
        alive = True

        while alive:
            alive = False

            for thread in threads:
                if thread.isAlive():
                    alive = True
                    thread.join(5)
    except KeyboardInterrupt:
        kb.threadContinue = False
        kb.threadException = True

        print
        logger.debug("waiting for threads to finish")

        warnMsg = "user aborted during common table existence check. "
        warnMsg += "sqlmap will display some tables only"
        logger.warn(warnMsg)

        try:
            while (threading.activeCount() > 1):
                pass

        except KeyboardInterrupt:
            raise sqlmapThreadException, "user aborted"
    finally:
        kb.bruteMode = False
        kb.threadContinue = True
        kb.threadException = False

    clearConsoleLine(True)
    dataToStdout("\n")

    if not retVal:
        warnMsg = "no table found"
        logger.warn(warnMsg)
    else:
        for item in retVal:
            if not kb.data.cachedTables.has_key(conf.db):
                kb.data.cachedTables[conf.db] = [item]
            else:
                kb.data.cachedTables[conf.db].append(item)

    return kb.data.cachedTables

def columnExists(columnFile, regex=None):
    if not conf.tbl:
        errMsg = "missing table parameter"
        raise sqlmapMissingMandatoryOptionException, errMsg

    columns = getFileItems(columnFile, unique=True)
    columns = filterListValue(columns, regex)

    if conf.db and not conf.db.endswith(METADB_SUFFIX):
        table = "%s%s%s" % (conf.db, '..' if Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE) else '.', conf.tbl)
    else:
        table = conf.tbl
    table = safeSQLIdentificatorNaming(table)

    retVal = []
    infoMsg = "checking column existence using items from '%s'" % columnFile
    logger.info(infoMsg)

    count = [0]
    length = len(columns)
    threads = []
    collock = threading.Lock()
    iolock = threading.Lock()
    kb.threadContinue = True
    kb.bruteMode = True

    def columnExistsThread():
        while count[0] < length and kb.threadContinue:
            collock.acquire()
            column = safeSQLIdentificatorNaming(columns[count[0]])
            count[0] += 1
            collock.release()

            result = inject.checkBooleanExpression("%s" % safeStringFormat("EXISTS(SELECT %s FROM %s)", (column, table)))

            iolock.acquire()

            if result:
                retVal.append(column)

                if conf.verbose in (1, 2):
                    clearConsoleLine(True)
                    infoMsg = "\r[%s] [INFO] retrieved: %s\n" % (time.strftime("%X"), column)
                    dataToStdout(infoMsg, True)

            if conf.verbose in (1, 2):
                status = '%d/%d items (%d%s)' % (count[0], length, round(100.0*count[0]/length), '%')
                dataToStdout("\r[%s] [INFO] tried %s" % (time.strftime("%X"), status), True)

            iolock.release()

    if conf.threads > 1:
        infoMsg = "starting %d threads" % conf.threads
        logger.info(infoMsg)
    else:
        message = "please enter number of threads? [Enter for %d (current)] " % conf.threads
        choice = readInput(message, default=str(conf.threads))
        if choice and choice.isdigit():
            conf.threads = int(choice)

    if conf.threads == 1:
        warnMsg = "running in a single-thread mode. This could take a while."
        logger.warn(warnMsg)

    # Start the threads
    for numThread in range(conf.threads):
        thread = threading.Thread(target=columnExistsThread, name=str(numThread))
        thread.start()
        threads.append(thread)

    # And wait for them to all finish
    try:
        alive = True

        while alive:
            alive = False

            for thread in threads:
                if thread.isAlive():
                    alive = True
                    thread.join(5)
    except KeyboardInterrupt:
        kb.threadContinue = False
        kb.threadException = True

        print
        logger.debug("waiting for threads to finish")

        warnMsg = "user aborted during common column existence check. "
        warnMsg += "sqlmap will display some columns only"
        logger.warn(warnMsg)

        try:
            while (threading.activeCount() > 1):
                pass

        except KeyboardInterrupt:
            raise sqlmapThreadException, "user aborted"
    finally:
        kb.bruteMode = False
        kb.threadContinue = True
        kb.threadException = False

    clearConsoleLine(True)
    dataToStdout("\n")

    if not retVal:
        warnMsg = "no column found"
        logger.warn(warnMsg)
    else:
        columns = {}

        for column in retVal:
            result = inject.checkBooleanExpression("%s" % safeStringFormat("EXISTS(SELECT %s FROM %s WHERE ROUND(%s)=ROUND(%s))", (column, table, column, column)))

            if result:
                columns[column] = 'numeric'
            else:
                columns[column] = 'non-numeric'

            dataToSessionFile("[%s][%s][%s][COLUMN_EXISTS][%s|%s %s]\n" % (conf.url, kb.injection.place,\
              safeFormatString(conf.parameters[kb.injection.place]), safeFormatString(table),\
              safeFormatString(column), safeFormatString(columns[column])))

        kb.data.cachedColumns[conf.db] = {conf.tbl: columns}

    return kb.data.cachedColumns
<|endoftext|>