<commit_before>import re
import urllib2
import gzip
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO
try:
    import simplejson
except ImportError:
    from django.utils import simplejson
from django.conf import settings
from django.utils.safestring import mark_safe
from oembed.models import ProviderRule, StoredOEmbed
from django.template.loader import render_to_string

END_OVERRIDES = (')', ',', '.', '>', ']', ';', '"', "'")
MAX_WIDTH = getattr(settings, "OEMBED_MAX_WIDTH", 320)
MAX_HEIGHT = getattr(settings, "OEMBED_MAX_HEIGHT", 240)
FORMAT = getattr(settings, "OEMBED_FORMAT", "json")

def fetch(url, user_agent="django-oembed/0.1"):
    """
    Fetches from a URL, respecting GZip encoding, etc.
    """
    request = urllib2.Request(url)
    request.add_header('User-Agent', user_agent)
    request.add_header('Accept-Encoding', 'gzip')
    opener = urllib2.build_opener()
    f = opener.open(request)
    result = f.read()
    if f.headers.get('content-encoding', '') == 'gzip':
        result = gzip.GzipFile(fileobj=StringIO(result)).read()
    f.close()
    return result

def re_parts(regex_list, text):
    """
    An iterator that returns the entire text, but split by which regex it
    matched, or none at all.  If it did, the first value of the returned tuple
    is the index into the regex list, otherwise -1.

    >>> first_re = re.compile('asdf')
    >>> second_re = re.compile('an')
    >>> list(re_parts([first_re, second_re], 'This is an asdf test.'))
    [(-1, 'This is '), (1, 'an'), (-1, ' '), (0, 'asdf'), (-1, ' test.')]

    >>> list(re_parts([first_re, second_re], 'asdfasdfasdf'))
    [(0, 'asdf'), (0, 'asdf'), (0, 'asdf')]

    >>> list(re_parts([], 'This is an asdf test.'))
    [(-1, 'This is an asdf test.')]

    >>> third_re = re.compile('sdf')
    >>> list(re_parts([first_re, second_re, third_re], 'This is an asdf test.'))
    [(-1, 'This is '), (1, 'an'), (-1, ' '), (0, 'asdf'), (-1, ' test.')]
    """
    def match_compare(x, y):
        return x.start() - y.start()
    prev_end = 0
    iters = [r.finditer(text) for r in regex_list]
    matches = []
    while iters:
        if matches:
            match = matches.pop(0)
            (start, end) = match.span()
            if start > prev_end:
                yield (-1, text[prev_end:start])
                yield (regex_list.index(match.re), text[start:end])
            elif start == prev_end:
                yield (regex_list.index(match.re), text[start:end])
            prev_end = end
        else:
            matches = []
            for iterator in iters:
                try:
                    matches.append(iterator.next())
                except StopIteration:
                    iters.remove(iterator)
            matches = sorted(matches, match_compare)
    last_bit = text[prev_end:]
    if len(last_bit) > 0:
        yield (-1, last_bit)

def replace(text, max_width=MAX_WIDTH, max_height=MAX_HEIGHT):
    """
    Scans a block of text, replacing anything matched by a ``ProviderRule``
    pattern with an OEmbed html snippet, if possible.

    Templates should be stored at oembed/{format}.html, so for example:

        oembed/video.html

    These templates are passed a context variable, ``response``, which is a
    dictionary representation of the response.
    """
    rules = list(ProviderRule.objects.all())
    patterns = [re.compile(r.regex) for r in rules] # Compiled patterns from the rules
    parts = [] # The parts that we will assemble into the final return value.
    indices = [] # List of indices of parts that need to be replaced with OEmbed stuff.
    indices_rules = [] # List of indices into the rules in order for which index was gotten by.
    urls = set() # A set of URLs to try to lookup from the database.
    stored = {} # A mapping of URLs to StoredOEmbed objects.
    index = 0
    # First we pass through the text, populating our data structures.
    for i, part in re_parts(patterns, text):
        if i == -1:
            parts.append(part)
            index += 1
        else:
            to_append = ""
            # If the link ends with one of our overrides, build a list
            while part[-1] in END_OVERRIDES:
                to_append += part[-1]
                part = part[:-1]
            indices.append(index)
            urls.add(part)
            indices_rules.append(i)
            parts.append(part)
            index += 1
            if to_append:
                parts.append(to_append)
                index += 1
    # Now we fetch a list of all stored patterns, and put it in a dictionary
    # mapping the URL to to the stored model instance.
    for stored_embed in StoredOEmbed.objects.filter(match__in=urls, max_width=max_width, max_height = max_height):
        stored[stored_embed.match] = stored_embed
    # Now we're going to do the actual replacement of URL to embed.
    for i, id_to_replace in enumerate(indices):
        rule = rules[indices_rules[i]]
        part = parts[id_to_replace]
        try:
            # Try to grab the stored model instance from our dictionary, and
            # use the stored HTML fragment as a replacement.
            parts[id_to_replace] = stored[part].html
        except KeyError:
            try:
                # Build the URL based on the properties defined in the OEmbed spec.
                url = u"%s?url=%s&maxwidth=%s&maxheight=%s&format=%s" % (
                    rule.endpoint, part, max_width, max_height, FORMAT
                )
                # Fetch the link and parse the JSON.
                resp = simplejson.loads(fetch(url))
                # Depending on the embed type, grab the associated template and
                # pass it the parsed JSON response as context.
                replacement = render_to_string('oembed/%s.html' % resp['type'], {'response': resp})
                if replacement:
                    stored_embed = StoredOEmbed.objects.create(
                        match = part,
                        max_width = max_width,
                        max_height = max_height,
                        html = replacement,
                    )
                    stored[stored_embed.match] = stored_embed
                    parts[id_to_replace] = replacement
                else:
                    raise ValueError
            except ValueError:
                parts[id_to_replace] = part
            except KeyError:
                parts[id_to_replace] = part
            except urllib2.HTTPError:
                parts[id_to_replace] = part
    # Combine the list into one string and return it.
    return mark_safe(u''.join(parts))
<commit_msg>Rolling back last commit, fixing it another way.<commit_after>import re
import urllib2
import gzip
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO
try:
    import simplejson
except ImportError:
    from django.utils import simplejson
from django.conf import settings
from django.utils.safestring import mark_safe
from oembed.models import ProviderRule, StoredOEmbed
from django.template.loader import render_to_string

END_OVERRIDES = (')', ',', '.', '>', ']', ';')
MAX_WIDTH = getattr(settings, "OEMBED_MAX_WIDTH", 320)
MAX_HEIGHT = getattr(settings, "OEMBED_MAX_HEIGHT", 240)
FORMAT = getattr(settings, "OEMBED_FORMAT", "json")

def fetch(url, user_agent="django-oembed/0.1"):
    """
    Fetches from a URL, respecting GZip encoding, etc.
    """
    request = urllib2.Request(url)
    request.add_header('User-Agent', user_agent)
    request.add_header('Accept-Encoding', 'gzip')
    opener = urllib2.build_opener()
    f = opener.open(request)
    result = f.read()
    if f.headers.get('content-encoding', '') == 'gzip':
        result = gzip.GzipFile(fileobj=StringIO(result)).read()
    f.close()
    return result

def re_parts(regex_list, text):
    """
    An iterator that returns the entire text, but split by which regex it
    matched, or none at all.  If it did, the first value of the returned tuple
    is the index into the regex list, otherwise -1.

    >>> first_re = re.compile('asdf')
    >>> second_re = re.compile('an')
    >>> list(re_parts([first_re, second_re], 'This is an asdf test.'))
    [(-1, 'This is '), (1, 'an'), (-1, ' '), (0, 'asdf'), (-1, ' test.')]

    >>> list(re_parts([first_re, second_re], 'asdfasdfasdf'))
    [(0, 'asdf'), (0, 'asdf'), (0, 'asdf')]

    >>> list(re_parts([], 'This is an asdf test.'))
    [(-1, 'This is an asdf test.')]

    >>> third_re = re.compile('sdf')
    >>> list(re_parts([first_re, second_re, third_re], 'This is an asdf test.'))
    [(-1, 'This is '), (1, 'an'), (-1, ' '), (0, 'asdf'), (-1, ' test.')]
    """
    def match_compare(x, y):
        return x.start() - y.start()
    prev_end = 0
    iters = [r.finditer(text) for r in regex_list]
    matches = []
    while iters:
        if matches:
            match = matches.pop(0)
            (start, end) = match.span()
            if start > prev_end:
                yield (-1, text[prev_end:start])
                yield (regex_list.index(match.re), text[start:end])
            elif start == prev_end:
                yield (regex_list.index(match.re), text[start:end])
            prev_end = end
        else:
            matches = []
            for iterator in iters:
                try:
                    matches.append(iterator.next())
                except StopIteration:
                    iters.remove(iterator)
            matches = sorted(matches, match_compare)
    last_bit = text[prev_end:]
    if len(last_bit) > 0:
        yield (-1, last_bit)

def replace(text, max_width=MAX_WIDTH, max_height=MAX_HEIGHT):
    """
    Scans a block of text, replacing anything matched by a ``ProviderRule``
    pattern with an OEmbed html snippet, if possible.

    Templates should be stored at oembed/{format}.html, so for example:

        oembed/video.html

    These templates are passed a context variable, ``response``, which is a
    dictionary representation of the response.
    """
    rules = list(ProviderRule.objects.all())
    patterns = [re.compile(r.regex) for r in rules] # Compiled patterns from the rules
    parts = [] # The parts that we will assemble into the final return value.
    indices = [] # List of indices of parts that need to be replaced with OEmbed stuff.
    indices_rules = [] # List of indices into the rules in order for which index was gotten by.
    urls = set() # A set of URLs to try to lookup from the database.
    stored = {} # A mapping of URLs to StoredOEmbed objects.
    index = 0
    # First we pass through the text, populating our data structures.
    for i, part in re_parts(patterns, text):
        if i == -1:
            parts.append(part)
            index += 1
        else:
            to_append = ""
            # If the link ends with one of our overrides, build a list
            while part[-1] in END_OVERRIDES:
                to_append += part[-1]
                part = part[:-1]
            indices.append(index)
            urls.add(part)
            indices_rules.append(i)
            parts.append(part)
            index += 1
            if to_append:
                parts.append(to_append)
                index += 1
    # Sanity checking to determine if the match is actually in another tag.
    for i, idx in enumerate(indices):
        if idx > 0:
            try:
                last_char = parts[idx-1][-1]
            except IndexError:
                last_char = ' '
            if last_char in "'\"":
                del indices[i]
        if idx < len(parts):
            try:
                first_char = parts[idx+1][0]
            except IndexError:
                first_char = ' '
            if first_char in "'\"":
                del indices[i]
    # Now we fetch a list of all stored patterns, and put it in a dictionary
    # mapping the URL to to the stored model instance.
    for stored_embed in StoredOEmbed.objects.filter(match__in=urls, max_width=max_width, max_height = max_height):
        stored[stored_embed.match] = stored_embed
    # Now we're going to do the actual replacement of URL to embed.
    for i, id_to_replace in enumerate(indices):
        rule = rules[indices_rules[i]]
        part = parts[id_to_replace]
        try:
            # Try to grab the stored model instance from our dictionary, and
            # use the stored HTML fragment as a replacement.
            parts[id_to_replace] = stored[part].html
        except KeyError:
            try:
                # Build the URL based on the properties defined in the OEmbed spec.
                url = u"%s?url=%s&maxwidth=%s&maxheight=%s&format=%s" % (
                    rule.endpoint, part, max_width, max_height, FORMAT
                )
                # Fetch the link and parse the JSON.
                resp = simplejson.loads(fetch(url))
                # Depending on the embed type, grab the associated template and
                # pass it the parsed JSON response as context.
                replacement = render_to_string('oembed/%s.html' % resp['type'], {'response': resp})
                if replacement:
                    stored_embed = StoredOEmbed.objects.create(
                        match = part,
                        max_width = max_width,
                        max_height = max_height,
                        html = replacement,
                    )
                    stored[stored_embed.match] = stored_embed
                    parts[id_to_replace] = replacement
                else:
                    raise ValueError
            except ValueError:
                parts[id_to_replace] = part
            except KeyError:
                parts[id_to_replace] = part
            except urllib2.HTTPError:
                parts[id_to_replace] = part
    # Combine the list into one string and return it.
    return mark_safe(u''.join(parts))
<|endoftext|>


<commit_before>from astropy.visualization.mpl_normalize import ImageNormalize
from astropy.io import fits
from astropy.wcs import WCS
#from rand_bkg import bkg_boxes
from astropy.convolution import Gaussian2DKernel
from astropy.stats import sigma_clipped_stats
from photutils.detection import detect_sources
from photutils.utils import random_cmap
from scipy.ndimage import binary_dilation
import pandas as pd
import numpy as np
import os
import glob
import odi_config as odi

def get_sdss_coords_offline(img, ota, inst,output='test.sdss'):
    hdulist = odi.fits.open(img)
    hdu = hdulist[ota]

    if inst == 'podi':
        pvlist = hdu.header['PV*']
        for pv in pvlist:
            tpv = 'T'+pv
            hdu.header.rename_keyword(pv, tpv, force=False)
    xdim = hdu.header['NAXIS1']
    ydim = hdu.header['NAXIS2']

    sdss_cat_img = hdulist['CAT.PHOTCALIB']
    sdss_cat_img_df = pd.DataFrame.from_dict(sdss_cat_img.data)
    hdulist.close()

    ota = float(ota.strip('OTA.SCI'))
    ota_matches_df = sdss_cat_img_df.iloc[np.where(sdss_cat_img_df['ODI_OTA'] == ota)]

    needed_columns = ['SDSS_RA','SDSS_DEC','SDSS_MAG_U',
                      'SDSS_ERR_U', u'SDSS_MAG_G', u'SDSS_ERR_G', u'SDSS_MAG_R',
                      'SDSS_ERR_R', u'SDSS_MAG_I', u'SDSS_ERR_I', u'SDSS_MAG_Z',
                      'SDSS_ERR_Z','ODI_OTA']

    output_df = ota_matches_df[needed_columns]
    output_df.to_csv(output,index=False)
    return xdim, ydim

def get_2mass_coords_offline(img, ota, inst,output='test.mass'):
    hdulist = odi.fits.open(img)
    hdu = hdulist[ota]

    if inst == 'podi':
        pvlist = hdu.header['PV*']
        for pv in pvlist:
            tpv = 'T'+pv
            hdu.header.rename_keyword(pv, tpv, force=False)
    xdim = hdu.header['NAXIS1']
    ydim = hdu.header['NAXIS2']

    mass_cat_img = hdulist['CAT.ODI+2MASS']
    mass_cat_img_df = mass_cat_img.data
    hdulist.close()

    ota = float(ota.strip('OTA.SCI'))
    ota_matches_df = mass_cat_img_df[np.where(mass_cat_img_df['OTA'] == ota)]

    junk_df = pd.DataFrame.from_dict(ota_matches_df)

    output_df = junk_df
    output_df.to_csv(output,index=False)
    return xdim, ydim

def main():
    pass

if __name__ == '__main__':
    main()
<commit_msg>catch missing OTA col from phot tables<commit_after>from astropy.visualization.mpl_normalize import ImageNormalize
from astropy.io import fits
from astropy.wcs import WCS
#from rand_bkg import bkg_boxes
from astropy.convolution import Gaussian2DKernel
from astropy.stats import sigma_clipped_stats
from photutils.detection import detect_sources
from photutils.utils import random_cmap
from scipy.ndimage import binary_dilation
import pandas as pd
import numpy as np
import os
import glob
from collections import OrderedDict
from astropy.io import fits

def get_sdss_coords_offline(img, ota, inst,output='test.sdss'):
    hdulist = fits.open(img)
    hdu = hdulist[ota]

    if inst == 'podi':
        pvlist = hdu.header['PV*']
        for pv in pvlist:
            tpv = 'T'+pv
            hdu.header.rename_keyword(pv, tpv, force=False)
    xdim = hdu.header['NAXIS1']
    ydim = hdu.header['NAXIS2']

    sdss_cat_img = hdulist['CAT.PHOTCALIB']
    sdss_cat_img_df = pd.DataFrame.from_dict(sdss_cat_img.data)

    ota = float(ota.strip('OTA.SCI'))
    try:
        ota_matches_df = sdss_cat_img_df.iloc[np.where(sdss_cat_img_df['ODI_OTA'] == ota)]
        needed_columns = ['SDSS_RA','SDSS_DEC','SDSS_MAG_U',
                          'SDSS_ERR_U', u'SDSS_MAG_G', u'SDSS_ERR_G', u'SDSS_MAG_R',
                          'SDSS_ERR_R', u'SDSS_MAG_I', u'SDSS_ERR_I', u'SDSS_MAG_Z',
                          'SDSS_ERR_Z','ODI_OTA']

        output_df = ota_matches_df[needed_columns]
        output_df.to_csv(output,index=False)
    except KeyError:
        oditable = hdulist['CAT.ODI'].data
        oditalbe_df = pd.DataFrame.from_dict(oditable)

        ODI_RA = np.squeeze(np.array(oditalbe_df['RA']))
        ODI_DEC = np.squeeze( np.array(oditalbe_df['DEC']))
        ODI_OTA = np.squeeze( np.array(oditalbe_df['OTA']))

        junkdict = OrderedDict([('ODI_RA',ODI_RA),
                                ('ODI_DEC',ODI_DEC),
                                ('ODI_OTA',ODI_OTA.astype(float))])
        junk_df = pd.DataFrame.from_dict(junkdict)

        matched_df = pd.merge(sdss_cat_img_df,junk_df ,on = ['ODI_RA','ODI_DEC'],how='inner')

        needed_columns = np.insert(sdss_cat_img_df.columns.values,0,'ODI_OTA')

        full_df = matched_df[needed_columns]
        print ota
        ota_matches_df = full_df.iloc[np.where(full_df['ODI_OTA'] == ota)]
        needed_columns = ['SDSS_RA','SDSS_DEC','SDSS_MAG_U',
                          'SDSS_ERR_U', u'SDSS_MAG_G', u'SDSS_ERR_G', u'SDSS_MAG_R',
                          'SDSS_ERR_R', u'SDSS_MAG_I', u'SDSS_ERR_I', u'SDSS_MAG_Z',
                          'SDSS_ERR_Z','ODI_OTA']
        output_df = full_df[needed_columns]
        output_df.to_csv(output,index=False)
    hdulist.close()
    return xdim, ydim

def get_2mass_coords_offline(img, ota, inst,output='test.mass'):
    hdulist = odi.fits.open(img)
    hdu = hdulist[ota]

    if inst == 'podi':
        pvlist = hdu.header['PV*']
        for pv in pvlist:
            tpv = 'T'+pv
            hdu.header.rename_keyword(pv, tpv, force=False)
    xdim = hdu.header['NAXIS1']
    ydim = hdu.header['NAXIS2']

    mass_cat_img = hdulist['CAT.ODI+2MASS']
    mass_cat_img_df = mass_cat_img.data
    hdulist.close()

    ota = float(ota.strip('OTA.SCI'))
    ota_matches_df = mass_cat_img_df[np.where(mass_cat_img_df['OTA'] == ota)]

    junk_df = pd.DataFrame.from_dict(ota_matches_df)

    output_df = junk_df
    output_df.to_csv(output,index=False)
    return xdim, ydim

def main():
    pass

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>#!/usr/bin/env python2
import sys, time, os
import re
import argparse
import xml.dom.minidom
from PIL import Image
from PIL import ImageEnhance
import pytesseract
from io import BytesIO
import pyscreenshot as ImageGrab
import getopt
from PyQt4 import QtCore, QtGui
from rdpy.protocol.rdp import rdp
from rdpy.ui.qt4 import RDPBitmapToQtImage
import rdpy.core.log as log
from rdpy.core.error import RDPSecurityNegoFail
from twisted.internet import task
import threading
import itertools

class colors:
    white = "\033[1;37m"
    normal = "\033[0;00m"
    red = "\033[1;31m"
    blue = "\033[1;34m"
    green = "\033[1;32m"
    lightblue = "\033[0;34m"

services = {}

def make_dic_gnmap():
    global services
    port = None
    with open(args.file, 'r') as nmap_file:
        for line in nmap_file:
            supported = ['ms-wbt-server']
            for name in supported:
                matches = re.compile(r'([0-9][0-9]*)/open/[a-z][a-z]*//' + name)
                try:
                    port =  matches.findall(line)[0]
                except:
                    continue

                ip = re.findall( r'[0-9]+(?:\.[0-9]+){3}', line)
                tmp_ports = matches.findall(line)
                for tmp_port in tmp_ports:
                        if name in services:
                            if tmp_port in services[name]:
                                services[name][tmp_port] += ip
                            else:
                                services[name][tmp_port] = ip
                        else:
                            services[name] = {tmp_port:ip}

def make_dic_xml():
    global loading
    global services
    doc = xml.dom.minidom.parse(args.file)
    supported = ['ms-wbt-server']
    for host in doc.getElementsByTagName("host"):
        try:
            address = host.getElementsByTagName("address")[0]
            ip = address.getAttribute("addr")
            eip = ip.encode("utf8")
            iplist = eip.split(',')
        except:
            # move to the next host
            continue
        try:
            status = host.getElementsByTagName("status")[0]
            state = status.getAttribute("state")
        except:
            state = ""
        try:
            ports = host.getElementsByTagName("ports")[0]
            ports = ports.getElementsByTagName("port")
        except:
            continue

        for port in ports:
            pn = port.getAttribute("portid")
            state_el = port.getElementsByTagName("state")[0]
            state = state_el.getAttribute("state")
            if state == "open":
                try:
                    service = port.getElementsByTagName("service")[0]
                    port_name = service.getAttribute("name")
                except:
                    service = ""
                    port_name = ""
                    product_descr = ""
                    product_ver = ""
                    product_extra = ""
                name = port_name.encode("utf-8")
                tmp_port = pn.encode("utf-8")
                if name in services:
                    if tmp_port in services[name]:
                        services[name][tmp_port] += iplist
                    else:
                        services[name][tmp_port] = iplist
                else:
                    services[name] = {tmp_port:iplist}

def loading():
    for c in itertools.cycle(['|', '/', '-', '\\']):
        if loading == True:
            break
        sys.stdout.write('\rTaking Screenshots Please Wait: ' + c)
        sys.stdout.flush()
        time.sleep(0.01)

# set log level
log._LOG_LEVEL = log.Level.WARNING

class RDPScreenShotFactory(rdp.ClientFactory):
    """
    @summary: Factory for screenshot exemple
    """
    __INSTANCE__ = 0
    __STATE__ = []

    def __init__(self, reactor, app, width, height, path, timeout):
        """
        @param reactor: twisted reactor
        @param width: {integer} width of screen
        @param height: {integer} height of screen
        @param path: {str} path of output screenshot
        @param timeout: {float} close connection after timeout s without any updating
        """
        RDPScreenShotFactory.__INSTANCE__ += 1
        self._reactor = reactor
        self._app = app
        self._width = width
        self._height = height
        self._path = path
        self._timeout = timeout
        #NLA server can't be screenshooting
        self._security = rdp.SecurityLevel.RDP_LEVEL_SSL

    def clientConnectionLost(self, connector, reason):
        """
        @summary: Connection lost event
        @param connector: twisted connector use for rdp connection (use reconnect to restart connection)
        @param reason: str use to advertise reason of lost connection
        """
        if reason.type == RDPSecurityNegoFail and self._security != "rdp":
            log.info("due to RDPSecurityNegoFail try standard security layer")
            self._security = rdp.SecurityLevel.RDP_LEVEL_RDP
            connector.connect()
            return

        log.info("connection lost : %s" % reason)
        RDPScreenShotFactory.__STATE__.append((connector.host, connector.port, reason))
        RDPScreenShotFactory.__INSTANCE__ -= 1
        if(RDPScreenShotFactory.__INSTANCE__ == 0):
            self._reactor.stop()
            self._app.exit()

    def clientConnectionFailed(self, connector, reason):
        """
        @summary: Connection failed event
        @param connector: twisted connector use for rdp connection (use reconnect to restart connection)
        @param reason: str use to advertise reason of lost connection
        """
        log.info("connection failed : %s"%reason)
        RDPScreenShotFactory.__STATE__.append((connector.host, connector.port, reason))
        RDPScreenShotFactory.__INSTANCE__ -= 1
        if(RDPScreenShotFactory.__INSTANCE__ == 0):
            self._reactor.stop()
            self._app.exit()

    def buildObserver(self, controller, addr):
        """
        @summary: build ScreenShot observer
        @param controller: RDPClientController
        @param addr: address of target
        """
        class ScreenShotObserver(rdp.RDPClientObserver):
            """
            @summary: observer that connect, cache every image received and save at deconnection
            """
            def __init__(self, controller, width, height, path, timeout, reactor):
                """
                @param controller: {RDPClientController}
                @param width: {integer} width of screen
                @param height: {integer} height of screen
                @param path: {str} path of output screenshot
                @param timeout: {float} close connection after timeout s without any updating
                @param reactor: twisted reactor
                """
                rdp.RDPClientObserver.__init__(self, controller)
                self._buffer = QtGui.QImage(width, height, QtGui.QImage.Format_RGB32)
                self._path = path
                self._timeout = timeout
                self._startTimeout = False
                self._reactor = reactor

            def onUpdate(self, destLeft, destTop, destRight, destBottom, width, height, bitsPerPixel, isCompress, data):
                """
                @summary: callback use when bitmap is received
                """
                image = RDPBitmapToQtImage(width, height, bitsPerPixel, isCompress, data);
                with QtGui.QPainter(self._buffer) as qp:
                # draw image
                    qp.drawImage(destLeft, destTop, image, 0, 0, destRight - destLeft + 1, destBottom - destTop + 1)
                if not self._startTimeout:
                    self._startTimeout = False
                    self._reactor.callLater(self._timeout, self.checkUpdate)

            def onReady(self):
                """
                @summary: callback use when RDP stack is connected (just before received bitmap)
                """
                log.info("connected %s" % addr)

            def onSessionReady(self):
                """
                @summary: Windows session is ready
                @see: rdp.RDPClientObserver.onSessionReady
                """
                pass

            def onClose(self):
                """
                @summary: callback use when RDP stack is closed
                """
                log.info("save screenshot into %s" % self._path)
                self._buffer.save(self._path)

            def checkUpdate(self):
                self._controller.close();

        controller.setScreen(self._width, self._height);
        controller.setSecurityLevel(self._security)
        return ScreenShotObserver(controller, self._width, self._height, self._path, self._timeout, self._reactor)

def main(width, height, path, timeout):
    """
    @summary: main algorithm
    @param height: {integer} height of screenshot
    @param width: {integer} width of screenshot
    @param timeout: {float} in sec
    @param hosts: {list(str(ip[:port]))}
    @return: {list(tuple(ip, port, Failure instance)} list of connection state
    """
    #create application
    app = QtGui.QApplication(sys.argv)

    #add qt4 reactor
    import qt4reactor
    qt4reactor.install()

    from twisted.internet import reactor

    with open(fname, 'r') as f:
        for ips in f:
            if ':' in ips:
                ip, port = ips.split(':')

            reactor.connectTCP(ip, int(port), RDPScreenShotFactory(reactor, app, width, height, path + "%s.jpg" % ip, timeout))

        reactor.runReturn()
        app.exec_()
        return RDPScreenShotFactory.__STATE__
    f.close()

def parse_args():

    parser = argparse.ArgumentParser(description=\

    "Usage: python brutemap.py <OPTIONS> \n")

    menu_group = parser.add_argument_group(colors.lightblue + 'Menu Options' + colors.normal)

    menu_group.add_argument('-f', '--file', help="GNMAP or XML file to parse", required=False)

    args = parser.parse_args()

    return args

args = parse_args()

if not os.path.exists("tmp/"):
    os.mkdir("tmp/")
tmppath = "tmp/"

width = 3072
height = 1536
path = "tmp/"
timeout = 5.0
bitsPerPixel = 24
Loading = False

try:
    doc = xml.dom.minidom.parse(args.file)
    make_dic_xml()
except:
    make_dic_gnmap()
if services is None:
    sys.exit(0)

t = threading.Thread(target=loading)
t.start()

for service in services:
    for port in services[service]:
        fname = 'tmp/'+service + '-' + port
        iplist = services[service][port]
        f = open(fname, 'w+')
        for ip in iplist:
            f.write(ip + ':' + port + '\n')
        f.close()

main(width, height, path, timeout)
'''
try:
    doc = xml.dom.minidom.parse(args.file)
    make_dic_xml()
except:
    make_dic_gnmap()

for service in services:
    for port in services[service]:
        iplist = services[service][port]
        for ip in iplist:
            print(ip)
'''
loading = True
#im = ImageGrab.grab()
#im2 = ImageGrab.grab_to_file('/root/Desktop/img22.png')
with open(fname, 'r') as fn:
    for fns in fn:
        ip, port = fns.split(':')
        img = Image.open('tmp/' + ip +'.jpg')
        #img = Image.open('tmp/' + ip +'.jpg').convert('L')
#        img = img.resize([int(2.4 * s) for s in img.size])
                #print('after resize')
#        enhancer = ImageEnhance.Sharpness(img)
        #enhancer = ImageEnhance.Contrast(img)
#        img = enhancer.enhance(0.7)
#        contrast = ImageEnhance.Contrast(img)
#        img = contrast.enhance(0.9)
#        img.save("tmp/test","jpeg")
#enhancer = ImageEnhance.Sharpness(img)
#img = enhancer.enhance(0.8)
#img.save('tmp.jpg')
#enhancer = ImageEnhance.Sharpness(img)
#img = enhancer.enhance(0.0)
#img.save('tmp.jpg')
        print "\nip address: " + ip + "\n"
        string = pytesseract.image_to_string(img)
        if '2012' in string:
            img = img.resize([int(2.4 * s) for s in img.size])
            enhancer = ImageEnhance.Sharpness(img)
            img = enhancer.enhance(0.7)
            contrast = ImageEnhance.Contrast(img)
            img = contrast.enhance(0.9)
            #color = ImageEnhance.Color(img)
            #img = color.enhance(0.)
            print(pytesseract.image_to_string(img))
            print "------------------------------------------------------------------------------------------------------------\n"
        else:
            img = img.resize([int(2.2 * s) for s in img.size])
            enhancer = ImageEnhance.Sharpness(img)
            img = enhancer.enhance(0.2)
            color = ImageEnhance.Color(img)
            img = color.enhance(0)
            #bright = ImageEnhance.Brightness(img)
            #img = bright.enhance(0.5)
            #contrast = ImageEnhance.Contrast(img)
            #img = contrast.enhance(0.2)
            print(pytesseract.image_to_string(img))
            print "------------------------------------------------------------------------------------------------------------\n"
<commit_msg>fixed no file error<commit_after>#!/usr/bin/env python2
import sys, time, os
import re
import argparse
import xml.dom.minidom
from PIL import Image
from PIL import ImageEnhance
import pytesseract
from io import BytesIO
import pyscreenshot as ImageGrab
import getopt
from PyQt4 import QtCore, QtGui
from rdpy.protocol.rdp import rdp
from rdpy.ui.qt4 import RDPBitmapToQtImage
import rdpy.core.log as log
from rdpy.core.error import RDPSecurityNegoFail
from twisted.internet import task
import threading
import itertools

class colors:
    white = "\033[1;37m"
    normal = "\033[0;00m"
    red = "\033[1;31m"
    blue = "\033[1;34m"
    green = "\033[1;32m"
    lightblue = "\033[0;34m"

services = {}

def make_dic_gnmap():
    global services
    port = None
    with open(args.file, 'r') as nmap_file:
        for line in nmap_file:
            supported = ['ms-wbt-server']
            for name in supported:
                matches = re.compile(r'([0-9][0-9]*)/open/[a-z][a-z]*//' + name)
                try:
                    port =  matches.findall(line)[0]
                except:
                    continue

                ip = re.findall( r'[0-9]+(?:\.[0-9]+){3}', line)
                tmp_ports = matches.findall(line)
                for tmp_port in tmp_ports:
                        if name in services:
                            if tmp_port in services[name]:
                                services[name][tmp_port] += ip
                            else:
                                services[name][tmp_port] = ip
                        else:
                            services[name] = {tmp_port:ip}

def make_dic_xml():
    global loading
    global services
    doc = xml.dom.minidom.parse(args.file)
    supported = ['ms-wbt-server']
    for host in doc.getElementsByTagName("host"):
        try:
            address = host.getElementsByTagName("address")[0]
            ip = address.getAttribute("addr")
            eip = ip.encode("utf8")
            iplist = eip.split(',')
        except:
            # move to the next host
            continue
        try:
            status = host.getElementsByTagName("status")[0]
            state = status.getAttribute("state")
        except:
            state = ""
        try:
            ports = host.getElementsByTagName("ports")[0]
            ports = ports.getElementsByTagName("port")
        except:
            continue

        for port in ports:
            pn = port.getAttribute("portid")
            state_el = port.getElementsByTagName("state")[0]
            state = state_el.getAttribute("state")
            if state == "open":
                try:
                    service = port.getElementsByTagName("service")[0]
                    port_name = service.getAttribute("name")
                except:
                    service = ""
                    port_name = ""
                    product_descr = ""
                    product_ver = ""
                    product_extra = ""
                name = port_name.encode("utf-8")
                tmp_port = pn.encode("utf-8")
                if name in services:
                    if tmp_port in services[name]:
                        services[name][tmp_port] += iplist
                    else:
                        services[name][tmp_port] = iplist
                else:
                    services[name] = {tmp_port:iplist}

def loading():
    for c in itertools.cycle(['|', '/', '-', '\\']):
        if loading == True:
            break
        sys.stdout.write('\rTaking Screenshots Please Wait: ' + c)
        sys.stdout.flush()
        time.sleep(0.01)

# set log level
log._LOG_LEVEL = log.Level.WARNING

class RDPScreenShotFactory(rdp.ClientFactory):
    """
    @summary: Factory for screenshot exemple
    """
    __INSTANCE__ = 0
    __STATE__ = []

    def __init__(self, reactor, app, width, height, path, timeout):
        """
        @param reactor: twisted reactor
        @param width: {integer} width of screen
        @param height: {integer} height of screen
        @param path: {str} path of output screenshot
        @param timeout: {float} close connection after timeout s without any updating
        """
        RDPScreenShotFactory.__INSTANCE__ += 1
        self._reactor = reactor
        self._app = app
        self._width = width
        self._height = height
        self._path = path
        self._timeout = timeout
        #NLA server can't be screenshooting
        self._security = rdp.SecurityLevel.RDP_LEVEL_SSL

    def clientConnectionLost(self, connector, reason):
        """
        @summary: Connection lost event
        @param connector: twisted connector use for rdp connection (use reconnect to restart connection)
        @param reason: str use to advertise reason of lost connection
        """
        if reason.type == RDPSecurityNegoFail and self._security != "rdp":
            log.info("due to RDPSecurityNegoFail try standard security layer")
            self._security = rdp.SecurityLevel.RDP_LEVEL_RDP
            connector.connect()
            return

        log.info("connection lost : %s" % reason)
        RDPScreenShotFactory.__STATE__.append((connector.host, connector.port, reason))
        RDPScreenShotFactory.__INSTANCE__ -= 1
        if(RDPScreenShotFactory.__INSTANCE__ == 0):
            self._reactor.stop()
            self._app.exit()

    def clientConnectionFailed(self, connector, reason):
        """
        @summary: Connection failed event
        @param connector: twisted connector use for rdp connection (use reconnect to restart connection)
        @param reason: str use to advertise reason of lost connection
        """
        log.info("connection failed : %s"%reason)
        RDPScreenShotFactory.__STATE__.append((connector.host, connector.port, reason))
        RDPScreenShotFactory.__INSTANCE__ -= 1
        if(RDPScreenShotFactory.__INSTANCE__ == 0):
            self._reactor.stop()
            self._app.exit()

    def buildObserver(self, controller, addr):
        """
        @summary: build ScreenShot observer
        @param controller: RDPClientController
        @param addr: address of target
        """
        class ScreenShotObserver(rdp.RDPClientObserver):
            """
            @summary: observer that connect, cache every image received and save at deconnection
            """
            def __init__(self, controller, width, height, path, timeout, reactor):
                """
                @param controller: {RDPClientController}
                @param width: {integer} width of screen
                @param height: {integer} height of screen
                @param path: {str} path of output screenshot
                @param timeout: {float} close connection after timeout s without any updating
                @param reactor: twisted reactor
                """
                rdp.RDPClientObserver.__init__(self, controller)
                self._buffer = QtGui.QImage(width, height, QtGui.QImage.Format_RGB32)
                self._path = path
                self._timeout = timeout
                self._startTimeout = False
                self._reactor = reactor

            def onUpdate(self, destLeft, destTop, destRight, destBottom, width, height, bitsPerPixel, isCompress, data):
                """
                @summary: callback use when bitmap is received
                """
                image = RDPBitmapToQtImage(width, height, bitsPerPixel, isCompress, data);
                with QtGui.QPainter(self._buffer) as qp:
                # draw image
                    qp.drawImage(destLeft, destTop, image, 0, 0, destRight - destLeft + 1, destBottom - destTop + 1)
                if not self._startTimeout:
                    self._startTimeout = False
                    self._reactor.callLater(self._timeout, self.checkUpdate)

            def onReady(self):
                """
                @summary: callback use when RDP stack is connected (just before received bitmap)
                """
                log.info("connected %s" % addr)

            def onSessionReady(self):
                """
                @summary: Windows session is ready
                @see: rdp.RDPClientObserver.onSessionReady
                """
                pass

            def onClose(self):
                """
                @summary: callback use when RDP stack is closed
                """
                log.info("save screenshot into %s" % self._path)
                self._buffer.save(self._path)

            def checkUpdate(self):
                self._controller.close();

        controller.setScreen(self._width, self._height);
        controller.setSecurityLevel(self._security)
        return ScreenShotObserver(controller, self._width, self._height, self._path, self._timeout, self._reactor)

def main(width, height, path, timeout):
    """
    @summary: main algorithm
    @param height: {integer} height of screenshot
    @param width: {integer} width of screenshot
    @param timeout: {float} in sec
    @param hosts: {list(str(ip[:port]))}
    @return: {list(tuple(ip, port, Failure instance)} list of connection state
    """
    #create application
    app = QtGui.QApplication(sys.argv)

    #add qt4 reactor
    import qt4reactor
    qt4reactor.install()

    from twisted.internet import reactor

    with open(fname, 'r') as f:
        for ips in f:
            if ':' in ips:
                ip, port = ips.split(':')

            reactor.connectTCP(ip, int(port), RDPScreenShotFactory(reactor, app, width, height, path + "%s.jpg" % ip, timeout))

        reactor.runReturn()
        app.exec_()
        return RDPScreenShotFactory.__STATE__
    f.close()

def parse_args():

    parser = argparse.ArgumentParser(description=\

    "Usage: python brutemap.py <OPTIONS> \n")

    menu_group = parser.add_argument_group(colors.lightblue + 'Menu Options' + colors.normal)

    menu_group.add_argument('-f', '--file', help="GNMAP or XML file to parse", required=True)

    args = parser.parse_args()

    return args

args = parse_args()

if not os.path.exists("tmp/"):
    os.mkdir("tmp/")
tmppath = "tmp/"

width = 3072
height = 1536
path = "tmp/"
timeout = 5.0
bitsPerPixel = 24
Loading = False

try:
    doc = xml.dom.minidom.parse(args.file)
    make_dic_xml()
except:
    make_dic_gnmap()
if services is None:
    sys.exit(0)

t = threading.Thread(target=loading)
t.start()

for service in services:
    for port in services[service]:
        fname = 'tmp/'+service + '-' + port
        iplist = services[service][port]
        f = open(fname, 'w+')
        for ip in iplist:
            f.write(ip + ':' + port + '\n')
        f.close()

main(width, height, path, timeout)
'''
try:
    doc = xml.dom.minidom.parse(args.file)
    make_dic_xml()
except:
    make_dic_gnmap()

for service in services:
    for port in services[service]:
        iplist = services[service][port]
        for ip in iplist:
            print(ip)
'''
loading = True
#im = ImageGrab.grab()
#im2 = ImageGrab.grab_to_file('/root/Desktop/img22.png')
with open(fname, 'r') as fn:
    for fns in fn:
        ip, port = fns.split(':')
        img = Image.open('tmp/' + ip +'.jpg')
        #img = Image.open('tmp/' + ip +'.jpg').convert('L')
#        img = img.resize([int(2.4 * s) for s in img.size])
                #print('after resize')
#        enhancer = ImageEnhance.Sharpness(img)
        #enhancer = ImageEnhance.Contrast(img)
#        img = enhancer.enhance(0.7)
#        contrast = ImageEnhance.Contrast(img)
#        img = contrast.enhance(0.9)
#        img.save("tmp/test","jpeg")
#enhancer = ImageEnhance.Sharpness(img)
#img = enhancer.enhance(0.8)
#img.save('tmp.jpg')
#enhancer = ImageEnhance.Sharpness(img)
#img = enhancer.enhance(0.0)
#img.save('tmp.jpg')
        print "\nip address: " + ip + "\n"
        string = pytesseract.image_to_string(img)
        if '2012' in string:
            img = img.resize([int(2.4 * s) for s in img.size])
            enhancer = ImageEnhance.Sharpness(img)
            img = enhancer.enhance(0.7)
            contrast = ImageEnhance.Contrast(img)
            img = contrast.enhance(0.9)
            #color = ImageEnhance.Color(img)
            #img = color.enhance(0.)
            print(pytesseract.image_to_string(img))
            print "------------------------------------------------------------------------------------------------------------\n"
        else:
            img = img.resize([int(2.2 * s) for s in img.size])
            enhancer = ImageEnhance.Sharpness(img)
            img = enhancer.enhance(0.2)
            color = ImageEnhance.Color(img)
            img = color.enhance(0)
            #bright = ImageEnhance.Brightness(img)
            #img = bright.enhance(0.5)
            #contrast = ImageEnhance.Contrast(img)
            #img = contrast.enhance(0.2)
            print(pytesseract.image_to_string(img))
            print "------------------------------------------------------------------------------------------------------------\n"
<|endoftext|>


<commit_before>import psycopg2, psycopg2.extras
import re

from . import dtuple

class ExpressionValue(str):
    pass

class ExpressionValueAdapter(object):
    def __init__(self, value):
        self.value = value

    def getquoted(self):
        return str(self.value)

psycopg2.extensions.register_adapter(ExpressionValue, ExpressionValueAdapter)

# Quoting of table/schema names in psycopg2:
# http://osdir.com/ml/python.db.psycopg.devel/2004-10/msg00012.html
class SchemaName(str):
    pass

class SchemaNameAdapter(object):
    def __init__(self, value):
        self.value = value

    def getquoted(self):
        return '"' + self.value.replace('"', '""') + '"'

psycopg2.extensions.register_adapter(SchemaName, SchemaNameAdapter)

# Use this for obtaining original psycopg2 tuple/list escaping behavior
class SqlArray(list):
    pass

#psycopg2.extensions.register_adapter(list, psycopg2.extensions.SQL_IN)
psycopg2.extensions.register_adapter(SqlArray, psycopg2._psycopg.List)

WHITESPACE_REGEXP = re.compile(r'^\s+', re.M)

class NotFoundError(StandardError):
    pass

def _lists_to_tuples(arg):
    if isinstance(arg, list):
        arg = tuple(arg)
    return arg

class CursorWrapper:
    def __init__(self, cursor, conn, debug=False):
        self.cursor = cursor
        self.conn = conn
        self._transaction_depth = 0
        self._transaction_depth_request = 0
        self._debug = debug

    def execute(self, sql, *args):
        return self.execute2(sql, args)

    def execute2(self, sql, args):
        sql = sql.replace('?', '%s')

        convert_lists = False

        if args is None:
            args = ()
        elif isinstance(args, dict):
            # keep as a dict
            # convert lists to tuples
            for key in args:
                value = args[key]
                if isinstance(value, list):
                    args[key] = tuple(value)
        elif isinstance(args, basestring) or getattr(args, '__len__', None) is None:
            args = (args,)
            convert_lists = True
        else:
            # a tuple or a list
            convert_lists = True

        if convert_lists:
            args = map(_lists_to_tuples, args)

        try:
            if self._debug:
                debug_sql = self.cursor.mogrify(sql, args)
                #debug_sql = sql.strip()
                #if args:
                    #debug_sql += ', ' + repr(args)
                debug_sql = WHITESPACE_REGEXP.sub('     ', debug_sql.strip())
                print 'SQL:', debug_sql
            return self.cursor.execute(sql, args)
        except psycopg2.OperationalError, e:
            if str(e).startswith('server closed the connection unexpectedly'):
                if self._transaction_depth == 0:
                    self.conn.reconnect()
                    self.cursor = self.conn.cursor()
                    self.cursor.execute(sql, args)
            else:
                raise

        if self._transaction_depth_request:
            self._transaction_depth += 1
            self._transaction_depth_request -= 1

    def execute_many(self, sql_commands):
        for sql in sql_commands.split(';'):
            sql = sql.strip()
            if sql:
                self.execute2(sql)

    #def fetchall(self):
        #return self.cursor.fetchall()

    # Higher-level interface

    # one/etc. take args as arguments.
    # to pass additional options use one2/etc.

    def one(self, sql, *args):
        return self.one2(sql, args)

    def one2(self, sql, args):
        self.execute2(sql, args)
        row = self.cursor.fetchone()
        if row is None:
            return None
        desc = dtuple.TupleDescriptor(self.cursor.description)
        row = dtuple.DatabaseTuple(desc, row)
        return row

    def one_check(self, sql, *args):
        return self.one_check2(sql, args)

    def one_check2(self, sql, args):
        row = self.one2(sql, args)
        if row is None:
            raise NotFoundError, "No data"
        return row

    def all(self, sql, *args):
        return self.all2(sql, args)

    def all2(self, sql, args):
        self.execute2(sql, args)
        desc = dtuple.TupleDescriptor(self.cursor.description)
        rows = self.cursor.fetchall()
        rows = [dtuple.DatabaseTuple(desc, row) for row in rows]
        return rows

    def one_value(self, sql, *args):
        return self.one_value2(sql, args)

    def one_value2(self, sql, args):
        self.execute2(sql, args)
        row = self.cursor.fetchone()
        if row is None:
            return None
        return row[0]

    def one_value_check(self, sql, *args):
        return self.one_value_check2(sql, args)

    def one_value_check2(self, sql, args):
        self.execute2(sql, args)
        row = self.cursor.fetchone()
        if row is None:
            raise NotFoundError, "No data"
        return row[0]

    def all_values(self, sql, *args):
        return self.all_values2(sql, args)

    def all_values2(self, sql, args):
        self.execute2(sql, args)
        desc = dtuple.TupleDescriptor(self.cursor.description)
        rows = self.cursor.fetchall()
        rows = [row[0] for row in rows]
        return rows

    # Transactions

    def begin(self):
        self._transaction_depth_request += 1

    def commit(self):
        self.conn.commit()
        self._transaction_depth -= 1

    def rollback(self):
        self.conn.rollback()
        self._transaction_depth -= 1

    def close(self):
        return self.cursor.close()

    # Statements

    #def insert_dict(self, table, dict):
    #    table = quote_name(table)
    #    keys_str = ','.join(map(quote_name, dict.keys()))
    #    values = []
    #    values_placeholders= []
    #    for value in dict.values():
    #        if isinstance(value, ExpressionValue):
    #            values_placeholders.append(value)
    #        else:
    #            values_placeholders.append('%s')
    #            values.append(value)
    #    values_str = ','.join(values_placeholders)
    #    sql = 'insert into %s (%s) values (%s)' % (table, keys_str, values_str)
    #    self.execute(sql, values)

    def insert_dict(self, table, dict):
        return self._insert_dict_impl(table, dict, False)

    def insert_dict_id(self, table, dict):
        return self._insert_dict_impl(table, dict, True)

    def _insert_dict_impl(self, table, dict, return_id):
        if not dict:
            raise ValueError, 'Cannot insert an empty dict'
        placeholders = ', '.join(['%s'] * len(dict))
        sql = 'insert into %%s (%s) values (%s)' % (placeholders, placeholders)
        if return_id:
            sql += ' returning (id)'
        table = SchemaName(table)
        columns = [SchemaName(column) for column in dict.keys()]
        values = dict.values()
        args = [table] + columns + values
        self.execute(sql, *args)
        if return_id:
            row = self.cursor.fetchone()
            return row[0]

    def update(self, table, attrs, conditions):
        if not attrs:
            raise ValueError, 'Trying to update with an empty attrs'
        placeholders = ', '.join(['%s=%s'] * len(attrs))
        sql = 'update %s set ' + placeholders
        args = [SchemaName(table)]
        for key in attrs:
            args.append(SchemaName(key))
            args.append(attrs[key])
        if conditions is not None:
            sql += ' where '
            if isinstance(conditions, str):
                sql += conditions
            elif isinstance(conditions, dict):
                if not conditions:
                    raise ValueError, 'Conditions was an empty dict'
                sql += ' and '.join(['%s=%s'] * len(conditions))
                for key in conditions:
                    args.append(SchemaName(key))
                    args.append(conditions[key])
            elif isinstance(conditions, tuple) or isinstance(conditions, list):
                sql += conditions[0]
                args += conditions[1:]
            else:
                raise ValueError, "Don't know what to do with these conditions"
        print sql, args
        self.execute(sql, *args)

    # DDL statements

    def add_fkey(self, table, column, target_table=None, target_column=None):
        target_table = column[:-3] + 's'
        target_column = 'id'
        name = '%s_%s_fk' % (table, column)
        vars = {
            'table': table,
            'column': column,
            'name': name,
            'target_table': target_table,
            'target_column': target_column,
        }
        self.execute('''
            alter table %(table)s add constraint %(name)s
                foreign key (%(column)s)
                references %(target_table)s (%(target_column)s)
        ''' % vars)

    def list_tables(self):
        return self.all_values('''
            select relname from pg_class
            where relkind='r' and
                relname not like %s
                and relname not like %s
        ''', 'pg_%', 'sql_%')

class CursorContextManager:
    def __init__(self, cursor):
        self.cursor = cursor

    def __enter__(self):
        return self.cursor

    def __exit__(self, type, value, traceback):
        self.cursor.close()

class TransactionalCursorContextManager(CursorContextManager):
    def __enter__(self):
        self.cursor.begin()
        return CursorContextManager.__enter__(self)

    def __exit__(self, type, value, traceback):
        if value is None:
            self.cursor.commit()
        else:
            self.cursor.rollback()
        return CursorContextManager.__exit__(self, type, value, traceback)

class ConnectionWrapper:
    def __init__(self, dsn, debug=False):
        self.dsn = dsn
        self.conn = None
        self._debug = debug

    def cursor(self):
        cursor = CursorWrapper(self.conn.cursor(), self, debug=self._debug)
        return CursorContextManager(cursor)

    def tx_cursor(self):
        cursor = CursorWrapper(self.conn.cursor(), self, debug=self._debug)
        return TransactionalCursorContextManager(cursor)

    # we need to rollback transactions after failed statements
    cursor = tx_cursor

    def commit(self):
        return self.conn.commit()

    def rollback(self):
        return self.conn.rollback()

    def connect(self):
        self.conn = psycopg2.connect(self.dsn)
        psycopg2.extras.register_hstore(self.conn)

    def reconnect(self):
        self.connect()

    def expr(self, value):
        return ExpressionValue(value)
<commit_msg>Derive classes from object to get better debugging information<commit_after>import psycopg2, psycopg2.extras
import re

from . import dtuple

class ExpressionValue(str):
    pass

class ExpressionValueAdapter(object):
    def __init__(self, value):
        self.value = value

    def getquoted(self):
        return str(self.value)

psycopg2.extensions.register_adapter(ExpressionValue, ExpressionValueAdapter)

# Quoting of table/schema names in psycopg2:
# http://osdir.com/ml/python.db.psycopg.devel/2004-10/msg00012.html
class SchemaName(str):
    pass

class SchemaNameAdapter(object):
    def __init__(self, value):
        self.value = value

    def getquoted(self):
        return '"' + self.value.replace('"', '""') + '"'

psycopg2.extensions.register_adapter(SchemaName, SchemaNameAdapter)

# Use this for obtaining original psycopg2 tuple/list escaping behavior
class SqlArray(list):
    pass

#psycopg2.extensions.register_adapter(list, psycopg2.extensions.SQL_IN)
psycopg2.extensions.register_adapter(SqlArray, psycopg2._psycopg.List)

WHITESPACE_REGEXP = re.compile(r'^\s+', re.M)

class NotFoundError(StandardError):
    pass

def _lists_to_tuples(arg):
    if isinstance(arg, list):
        arg = tuple(arg)
    return arg

class CursorWrapper(object):
    def __init__(self, cursor, conn, debug=False):
        self.cursor = cursor
        self.conn = conn
        self._transaction_depth = 0
        self._transaction_depth_request = 0
        self._debug = debug

    def execute(self, sql, *args):
        return self.execute2(sql, args)

    def execute2(self, sql, args):
        sql = sql.replace('?', '%s')

        convert_lists = False

        if args is None:
            args = ()
        elif isinstance(args, dict):
            # keep as a dict
            # convert lists to tuples
            for key in args:
                value = args[key]
                if isinstance(value, list):
                    args[key] = tuple(value)
        elif isinstance(args, basestring) or getattr(args, '__len__', None) is None:
            args = (args,)
            convert_lists = True
        else:
            # a tuple or a list
            convert_lists = True

        if convert_lists:
            args = map(_lists_to_tuples, args)

        try:
            if self._debug:
                debug_sql = self.cursor.mogrify(sql, args)
                #debug_sql = sql.strip()
                #if args:
                    #debug_sql += ', ' + repr(args)
                debug_sql = WHITESPACE_REGEXP.sub('     ', debug_sql.strip())
                print 'SQL:', debug_sql
            return self.cursor.execute(sql, args)
        except psycopg2.OperationalError, e:
            if str(e).startswith('server closed the connection unexpectedly'):
                if self._transaction_depth == 0:
                    self.conn.reconnect()
                    self.cursor = self.conn.cursor()
                    self.cursor.execute(sql, args)
            else:
                raise

        if self._transaction_depth_request:
            self._transaction_depth += 1
            self._transaction_depth_request -= 1

    def execute_many(self, sql_commands):
        for sql in sql_commands.split(';'):
            sql = sql.strip()
            if sql:
                self.execute2(sql)

    #def fetchall(self):
        #return self.cursor.fetchall()

    # Higher-level interface

    # one/etc. take args as arguments.
    # to pass additional options use one2/etc.

    def one(self, sql, *args):
        return self.one2(sql, args)

    def one2(self, sql, args):
        self.execute2(sql, args)
        row = self.cursor.fetchone()
        if row is None:
            return None
        desc = dtuple.TupleDescriptor(self.cursor.description)
        row = dtuple.DatabaseTuple(desc, row)
        return row

    def one_check(self, sql, *args):
        return self.one_check2(sql, args)

    def one_check2(self, sql, args):
        row = self.one2(sql, args)
        if row is None:
            raise NotFoundError, "No data"
        return row

    def all(self, sql, *args):
        return self.all2(sql, args)

    def all2(self, sql, args):
        self.execute2(sql, args)
        desc = dtuple.TupleDescriptor(self.cursor.description)
        rows = self.cursor.fetchall()
        rows = [dtuple.DatabaseTuple(desc, row) for row in rows]
        return rows

    def one_value(self, sql, *args):
        return self.one_value2(sql, args)

    def one_value2(self, sql, args):
        self.execute2(sql, args)
        row = self.cursor.fetchone()
        if row is None:
            return None
        return row[0]

    def one_value_check(self, sql, *args):
        return self.one_value_check2(sql, args)

    def one_value_check2(self, sql, args):
        self.execute2(sql, args)
        row = self.cursor.fetchone()
        if row is None:
            raise NotFoundError, "No data"
        return row[0]

    def all_values(self, sql, *args):
        return self.all_values2(sql, args)

    def all_values2(self, sql, args):
        self.execute2(sql, args)
        desc = dtuple.TupleDescriptor(self.cursor.description)
        rows = self.cursor.fetchall()
        rows = [row[0] for row in rows]
        return rows

    # Transactions

    def begin(self):
        self._transaction_depth_request += 1

    def commit(self):
        self.conn.commit()
        self._transaction_depth -= 1

    def rollback(self):
        self.conn.rollback()
        self._transaction_depth -= 1

    def close(self):
        return self.cursor.close()

    # Statements

    #def insert_dict(self, table, dict):
    #    table = quote_name(table)
    #    keys_str = ','.join(map(quote_name, dict.keys()))
    #    values = []
    #    values_placeholders= []
    #    for value in dict.values():
    #        if isinstance(value, ExpressionValue):
    #            values_placeholders.append(value)
    #        else:
    #            values_placeholders.append('%s')
    #            values.append(value)
    #    values_str = ','.join(values_placeholders)
    #    sql = 'insert into %s (%s) values (%s)' % (table, keys_str, values_str)
    #    self.execute(sql, values)

    def insert_dict(self, table, dict):
        return self._insert_dict_impl(table, dict, False)

    def insert_dict_id(self, table, dict):
        return self._insert_dict_impl(table, dict, True)

    def _insert_dict_impl(self, table, dict, return_id):
        if not dict:
            raise ValueError, 'Cannot insert an empty dict'
        placeholders = ', '.join(['%s'] * len(dict))
        sql = 'insert into %%s (%s) values (%s)' % (placeholders, placeholders)
        if return_id:
            sql += ' returning (id)'
        table = SchemaName(table)
        columns = [SchemaName(column) for column in dict.keys()]
        values = dict.values()
        args = [table] + columns + values
        self.execute(sql, *args)
        if return_id:
            row = self.cursor.fetchone()
            return row[0]

    def update(self, table, attrs, conditions):
        if not attrs:
            raise ValueError, 'Trying to update with an empty attrs'
        placeholders = ', '.join(['%s=%s'] * len(attrs))
        sql = 'update %s set ' + placeholders
        args = [SchemaName(table)]
        for key in attrs:
            args.append(SchemaName(key))
            args.append(attrs[key])
        if conditions is not None:
            sql += ' where '
            if isinstance(conditions, str):
                sql += conditions
            elif isinstance(conditions, dict):
                if not conditions:
                    raise ValueError, 'Conditions was an empty dict'
                sql += ' and '.join(['%s=%s'] * len(conditions))
                for key in conditions:
                    args.append(SchemaName(key))
                    args.append(conditions[key])
            elif isinstance(conditions, tuple) or isinstance(conditions, list):
                sql += conditions[0]
                args += conditions[1:]
            else:
                raise ValueError, "Don't know what to do with these conditions"
        print sql, args
        self.execute(sql, *args)

    # DDL statements

    def add_fkey(self, table, column, target_table=None, target_column=None):
        target_table = column[:-3] + 's'
        target_column = 'id'
        name = '%s_%s_fk' % (table, column)
        vars = {
            'table': table,
            'column': column,
            'name': name,
            'target_table': target_table,
            'target_column': target_column,
        }
        self.execute('''
            alter table %(table)s add constraint %(name)s
                foreign key (%(column)s)
                references %(target_table)s (%(target_column)s)
        ''' % vars)

    def list_tables(self):
        return self.all_values('''
            select relname from pg_class
            where relkind='r' and
                relname not like %s
                and relname not like %s
        ''', 'pg_%', 'sql_%')

class CursorContextManager(object):
    def __init__(self, cursor):
        self.cursor = cursor

    def __enter__(self):
        return self.cursor

    def __exit__(self, type, value, traceback):
        self.cursor.close()

class TransactionalCursorContextManager(CursorContextManager):
    def __enter__(self):
        self.cursor.begin()
        return CursorContextManager.__enter__(self)

    def __exit__(self, type, value, traceback):
        if value is None:
            self.cursor.commit()
        else:
            self.cursor.rollback()
        return CursorContextManager.__exit__(self, type, value, traceback)

class ConnectionWrapper(object):
    def __init__(self, dsn, debug=False):
        self.dsn = dsn
        self.conn = None
        self._debug = debug

    def cursor(self):
        cursor = CursorWrapper(self.conn.cursor(), self, debug=self._debug)
        return CursorContextManager(cursor)

    def tx_cursor(self):
        cursor = CursorWrapper(self.conn.cursor(), self, debug=self._debug)
        return TransactionalCursorContextManager(cursor)

    # we need to rollback transactions after failed statements
    cursor = tx_cursor

    def commit(self):
        return self.conn.commit()

    def rollback(self):
        return self.conn.rollback()

    def connect(self):
        self.conn = psycopg2.connect(self.dsn)
        psycopg2.extras.register_hstore(self.conn)

    def reconnect(self):
        self.connect()

    def expr(self, value):
        return ExpressionValue(value)
<|endoftext|>


<commit_before># -#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#

import miscIO
import tsvIO

import math
import sys

# -#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#

NA_VALUE = -999999

# -#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#

def lookAtHeaderTokens(aTokens):

    patientList = []
    typeDict = {}

    for a in aTokens:
        a = a.lower()
        if (a.startswith("tcga-")):
            patientID = a[8:12]
            if (patientID not in patientList):
                patientList += [patientID]
            if (len(a) >= 15):
                typeID = a[13:15]
                if (typeID not in typeDict.keys()):
                    typeDict[typeID] = 0
                typeDict[typeID] += 1
            else:
                print " WARNING : no typeID ??? <%s> " % a

    if (len(patientList) > 0):
        print " "
        print " # of unique patients : ", len(patientList)
        print " sample type counts   : ", typeDict
        print " "
        print " "

# -#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#

def lookAtLine(aLine):

    if (1):
        print len(aLine)

        numTab = 0
        numLF = 0
        numCR = 0
        numSpace = 0
        numDigit = 0
        numLetter = 0

        numLinesOut = 0

        i1 = 0
        for ii in range(len(aLine)):
            ordVal = ord(aLine[ii])
            if (1):
                if (ordVal == 9):
                    # this is a tab ...
                    numTab += 1
                elif (ordVal == 10):
                    numLF += 1
                elif (ordVal == 13):
                    numCR += 1
                elif (ordVal == 32):
                    numSpace += 1
                elif ((ordVal >= 48 and ordVal <= 57) or (ordVal == 46)):
                    numDigit += 1
                elif ((ordVal >= 65 and ordVal <= 90) or (ordVal >= 97 and ordVal <= 122)):
                    numLetter += 1
                elif (ordVal < 32 or ordVal > 126):
                    print " %6d     %3d " % (ii, ordVal)
                else:

                    # print " %6d <%s> %3d " % ( ii, aLine[ii], ord ( aLine[ii]
                    # ) )
                    doNothing = 1
            if (ordVal == 13):
                i2 = ii
                # print " --> writing out from %d to %d " % ( i1, i2 )
                # print " <%s> " % aLine[i1:i2]
                numLinesOut += 1
                ## if ( numLinesOut == 5 ): sys.exit(-1)
                ## fhOut.write ( "%s\n" % aLine[i1:i2] )
                i1 = i2 + 1

        print numTab, numLF, numCR, numSpace, numDigit, numLetter
        print numLinesOut

    sys.exit(-1)

# -#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#

def findIndex(rowLabels, aLabel):

    for ii in range(len(rowLabels)):
        if (rowLabels[ii].lower() == aLabel.lower()):
            return (ii)

    if (0):
        print " ??? not found ??? ", aLabel
        print rowLabels[:5]
        print rowLabels[-5:]
    return (-1)

# -#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#

if __name__ == "__main__":

    if (len(sys.argv) != 3):
        print ' Usage : %s <input TSV #1> <input TSV #2> ' % sys.argv[0]
        sys.exit(-1)

    inFileA = sys.argv[1]
    inFileB = sys.argv[2]

    try:
        fhA = file(inFileA, 'r')
        fhA.close()
        fhB = file(inFileB, 'r')
        fhB.close()
    except:
        print " one or the other file does not exist ??? "
        print inFileA
        print inFileB
        sys.exit(-1)

    print " "
    print " reading input files ... : "
    print "     file A : ", inFileA
    print "     file B : ", inFileB
    print " "

    dataA = tsvIO.readTSV(inFileA)
    dataB = tsvIO.readTSV(inFileB)

    if (len(dataA) == 0):
        print " input file A does not exist ??? "
        print inFileA
        sys.exit(-1)

    if (len(dataB) == 0):
        print inFileB
        print " input file B does not exist ??? "
        sys.exit(-1)

    # first take a look at the feature (row) labels ...

    rowLabelsA = []
    for aLabel in dataA['rowLabels']:
        rowLabelsA += [aLabel.lower()]
    rowLabelsB = []
    for aLabel in dataB['rowLabels']:
        rowLabelsB += [aLabel.lower()]

    numAinB = 0
    featLabelsAnotB = []
    for aLabel in rowLabelsA:
        if (aLabel in rowLabelsB):
            numAinB += 1
        else:
            featLabelsAnotB += [aLabel]

    numBinA = 0
    featLabelsBnotA = []
    for aLabel in rowLabelsB:
        if (aLabel in rowLabelsA):
            numBinA += 1
        else:
            featLabelsBnotA += [aLabel]

    print " "
    print "     --> total # of features in each : %6d  %6d " % (len(rowLabelsA), len(rowLabelsB))
    print "         overlap counts              : %6d  %6d " % (numAinB, numBinA)

    featLabelsAnotB.sort()
    featLabelsBnotA.sort()

    print " "
    if (len(featLabelsAnotB) > 0):
        print " %d labels in A but not in B (string names have been arbitrarily forced to lower-case) : " % (len(featLabelsAnotB))
        for aLabel in featLabelsAnotB:
            print "     feat in A not B: ", aLabel
    else:
        print " NO labels in A but not in B (string names have been arbitrarily forced to lower-case) : "

    print " "
    if (len(featLabelsBnotA) > 0):
        print " %d labels in B but not in A (string names have been arbitrarily forced to lower-case) : " % (len(featLabelsBnotA))
        for aLabel in featLabelsBnotA:
            print "     feat in B not A: ", aLabel
    else:
        print " NO labels in B but not in A (string names have been arbitrarily forced to lower-case) : "

    # and next at the column (sample) labels ...

    # first check the sample ID lengths ...
    colLabelA_len = len(dataA['colLabels'][0])
    colLabelB_len = len(dataB['colLabels'][0])
    minLen = min(colLabelA_len, colLabelB_len)
    if (minLen < colLabelA_len):
        print " NOTE: sample IDs in A will be abbreviated ... eg from <%s> to <%s> " \
            % (dataA['colLabels'][0], dataA['colLabels'][0][:minLen])
    if (minLen < colLabelB_len):
        print " NOTE: sample IDs in B will be abbreviated ... eg from <%s> to <%s> " \
            % (dataB['colLabels'][0], dataB['colLabels'][0][:minLen])

    colLabelsA = []
    for aLabel in dataA['colLabels']:
        colLabelsA += [aLabel.lower()[:minLen]]
    colLabelsB = []
    for aLabel in dataB['colLabels']:
        colLabelsB += [aLabel.lower()[:minLen]]

    numAinB = 0
    samplesAnotB = []
    for aLabel in colLabelsA:
        if (aLabel in colLabelsB):
            numAinB += 1
        else:
            samplesAnotB += [aLabel]

    numBinA = 0
    samplesBnotA = []
    for aLabel in colLabelsB:
        if (aLabel in colLabelsA):
            numBinA += 1
        else:
            samplesBnotA += [aLabel]

    print " "
    print "     --> total # of samples in each  : %6d  %6d " % (len(colLabelsA), len(colLabelsB))
    print "         overlap counts              : %6d  %6d " % (numAinB, numBinA)

    samplesAnotB.sort()
    samplesBnotA.sort()

    print " "
    if (len(samplesAnotB) > 0):
        print " %d samples in A but not in B (string names have been arbitrarily forced to lower-case) : " % (len(samplesAnotB))
        for aLabel in samplesAnotB:
            print aLabel
    else:
        print " NO samples in A but not in B (string names have been arbitrarily forced to lower-case) : "

    print " "
    if (len(samplesBnotA) > 0):
        print " %d samples in B but not in A (string names have been arbitrarily forced to lower-case) : " % (len(samplesBnotA))
        for aLabel in samplesBnotA:
            print aLabel
    else:
        print " NO samples in B but not in A (string names have been arbitrarily forced to lower-case) : "

    # now, for the features that exist in both, compare the *data* values ...
    print " "
    print " "
    print " "
    print " now checking data values, feature by feature ... "
    print " (based on features in A only) "

    for aLabel in rowLabelsA:

        # skip this feature if it is in the A-not-B list ...
        if (aLabel in featLabelsAnotB):
            continue

        # skip binary indicator features since they are generated automatically
        # ...
        if (aLabel[6:9] == ":i("):
            continue

        # get the feature index for each matrix ...
        iA = findIndex(dataA['rowLabels'], aLabel)
        iB = findIndex(dataB['rowLabels'], aLabel)
        # print aLabel, iA, iB

        # grab the data vector from each matrix ...
        dataVecA = dataA['dataMatrix'][iA]
        dataVecB = dataB['dataMatrix'][iB]
        # print dataVecA, dataVecB

        numDiff = 0
        numNotNA = 0

        sumDiff = 0.
        sumDiff2 = 0.
        sumN = 0

        # now loop over the individual data values for this feature
        for jA in range(len(dataVecA)):
            sampleA = dataA['colLabels'][jA]

            if (str(dataVecA[jA]) == "NA" or str(dataVecA[jA]) == "-999999"):
                continue

            else:
                numNotNA += 1

                jB = findIndex(dataB['colLabels'], sampleA)
                if (jB < 0):
                    # print " sample <%s> not found in matrix B " % sampleA
                    if (str(dataVecA[jA]) == "NA"):
                        doNothing = 1
                    elif (str(dataVecA[jA]) == "-999999"):
                        doNothing = 1
                    else:
                        numDiff += 1
                else:
                    # print jA, jB

                    if (str(dataVecB[jB]) == "NA" or str(dataVecB[jB]) == "-999999"):
                        doNothing = 1

                    elif (str(dataVecA[jA]).lower() != str(dataVecB[jB]).lower()):
                        # print " values are different for sample <%s> : <%s>
                        # vs <%s> " % ( sampleA, str(dataVecA[jA]),
                        # str(dataVecB[jB]) )
                        numDiff += 1
                        try:
                            diffVal = float (dataVecA[jA] ) - \
                                float(dataVecB[jB])
                            if (abs(diffVal) > 99999):
                                print " ERROR ??? using NA value ??? !!! ", jA, dataVecA[jA], jB, dataVecB[jB]
                            sumDiff += diffVal
                            sumDiff2 += (diffVal * diffVal)
                            sumN += 1
                        except:
                            doNothing = 1

        if (numDiff == 0):
            print " all %4d non-NA values are the same <%s> (%d) " % (numNotNA, aLabel, len(dataVecA))
        else:
            if (sumN > 3):
                meanDiff = float(sumDiff) / float(sumN)
                meanDiff2 = float(sumDiff2) / float(sumN)
                sigmDiff = math.sqrt(
                    max((meanDiff2 - (meanDiff * meanDiff)), 0.))
                if (abs(meanDiff) > 0.02 or sigmDiff > 0.02):
                    print " %4d out of %4d non-NA values are different <%s> (%d) [%d,%.2f,%.2f] " % \
                        (numDiff, numNotNA, aLabel,
                         len(dataVecA), sumN, meanDiff, sigmDiff)
                else:
                    print " %4d out of %4d non-NA values are different <%s> (%d) -- but not significantly [%d,%.2f,%.2f] " % \
                        (numDiff, numNotNA, aLabel,
                         len(dataVecA), sumN, meanDiff, sigmDiff)
            else:
                print " %4d out of %4d non-NA values are different <%s> (%d) " % (numDiff, numNotNA, aLabel, len(dataVecA))

    print " "
    print " FINISHED "
    print " "

# -#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#
<commit_msg>modifications for case-insensitive handling of feature names<commit_after># -#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#

import miscIO
import tsvIO

import math
import sys

# -#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#

NA_VALUE = -999999

# -#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#

def lookAtHeaderTokens(aTokens):

    patientList = []
    typeDict = {}

    for a in aTokens:
        if (a.upper().startswith("TCGA-")):
            patientID = a[8:12].upper()
            if (patientID not in patientList):
                patientList += [patientID]
            if (len(a) >= 15):
                typeID = a[13:15]
                if (typeID not in typeDict.keys()):
                    typeDict[typeID] = 0
                typeDict[typeID] += 1
            else:
                print " WARNING : no typeID ??? <%s> " % a

    if (len(patientList) > 0):
        print " "
        print " # of unique patients : ", len(patientList)
        print " sample type counts   : ", typeDict
        print " "
        print " "

# -#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#

def lookAtLine(aLine):

    if (1):
        print len(aLine)

        numTab = 0
        numLF = 0
        numCR = 0
        numSpace = 0
        numDigit = 0
        numLetter = 0

        numLinesOut = 0

        i1 = 0
        for ii in range(len(aLine)):
            ordVal = ord(aLine[ii])
            if (1):
                if (ordVal == 9):
                    # this is a tab ...
                    numTab += 1
                elif (ordVal == 10):
                    numLF += 1
                elif (ordVal == 13):
                    numCR += 1
                elif (ordVal == 32):
                    numSpace += 1
                elif ((ordVal >= 48 and ordVal <= 57) or (ordVal == 46)):
                    numDigit += 1
                elif ((ordVal >= 65 and ordVal <= 90) or (ordVal >= 97 and ordVal <= 122)):
                    numLetter += 1
                elif (ordVal < 32 or ordVal > 126):
                    print " %6d     %3d " % (ii, ordVal)
                else:

                    # print " %6d <%s> %3d " % ( ii, aLine[ii], ord ( aLine[ii]
                    # ) )
                    doNothing = 1
            if (ordVal == 13):
                i2 = ii
                # print " --> writing out from %d to %d " % ( i1, i2 )
                # print " <%s> " % aLine[i1:i2]
                numLinesOut += 1
                ## if ( numLinesOut == 5 ): sys.exit(-1)
                ## fhOut.write ( "%s\n" % aLine[i1:i2] )
                i1 = i2 + 1

        print numTab, numLF, numCR, numSpace, numDigit, numLetter
        print numLinesOut

    sys.exit(-1)

# -#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#

def findIndex(rowLabels, aLabel):

    ## print " in findIndex ... ", aLabel
    ## print rowLabels[:5]

    if ( aLabel[1]==':' and aLabel[6]==':' ):
        if ( aLabel[2:6].lower()=="clin"  or  aLabel[2:6].lower()=="samp" ):
            aLabel2 = aLabel[6:].lower()
        else:
            aLabel2 = aLabel.lower()
    else:
        aLabel2 = aLabel.lower()

    for ii in range(len(rowLabels)):

        rLabel = rowLabels[ii]
        if ( rLabel[1]==':' and rLabel[6]==':' ):
            if ( rLabel[2:6].lower()=="clin"  or  rLabel[2:6].lower()=="samp" ):
                rLabel2 = rLabel[6:].lower()
            else:
                rLabel2 = rLabel.lower()
        else:
            rLabel2 = rLabel.lower()

        ## print "         comparing <%s> and <%s> " % ( rLabel2, aLabel2 )
        if ( rLabel2 == aLabel2 ):
            return (ii)

    if (0):
        print " ??? not found ??? ", aLabel
        print rowLabels[:5]
        print rowLabels[-5:]
    return (-1)

# -#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#

if __name__ == "__main__":

    if (len(sys.argv) != 3):
        print ' Usage : %s <input TSV #1> <input TSV #2> ' % sys.argv[0]
        sys.exit(-1)

    inFileA = sys.argv[1]
    inFileB = sys.argv[2]

    try:
        fhA = file(inFileA, 'r')
        fhA.close()
        fhB = file(inFileB, 'r')
        fhB.close()
    except:
        print " one or the other file does not exist ??? "
        print inFileA
        print inFileB
        sys.exit(-1)

    print " "
    print " reading input files ... : "
    print "     file A : ", inFileA
    print "     file B : ", inFileB
    print " "

    dataA = tsvIO.readTSV(inFileA)
    dataB = tsvIO.readTSV(inFileB)

    if (len(dataA) == 0):
        print " input file A does not exist ??? "
        print inFileA
        sys.exit(-1)

    if (len(dataB) == 0):
        print inFileB
        print " input file B does not exist ??? "
        sys.exit(-1)

    # first take a look at the feature (row) labels ...

    rowLabelsA = []
    for aLabel in dataA['rowLabels']:
        aTokens = aLabel.split(':')
        if ( len(aTokens) > 3 ):
            if ( aTokens[1].lower()=="clin"  or  aTokens[1].lower()=="samp" ):
                rowLabelsA += [ aLabel[6:].lower() ]
            else:
                rowLabelsA += [ aLabel.lower() ]
        else:
            rowLabelsA += [ aLabel.lower() ]
    rowLabelsA.sort()

    rowLabelsB = []
    for bLabel in dataB['rowLabels']:
        bTokens = bLabel.split(':')
        if ( len(bTokens) > 3 ):
            if ( bTokens[1].lower()=="clin"  or  bTokens[1].lower()=="samp" ):
                rowLabelsB += [ bLabel[6:].lower() ]
            else:
                rowLabelsB += [ bLabel.lower() ]
        else:
            rowLabelsB += [ bLabel.lower() ]
    rowLabelsB.sort()

    ## print len(rowLabelsA), len(rowLabelsB)
    ## print rowLabelsA[:5], rowLabelsA[-5:]
    ## print rowLabelsB[:5], rowLabelsB[-5:]

    numAinB = 0
    featLabelsAnotB = []
    for aLabel in rowLabelsA:
        if (aLabel in rowLabelsB):
            numAinB += 1
        else:
            featLabelsAnotB += [aLabel]

    numBinA = 0
    featLabelsBnotA = []
    for aLabel in rowLabelsB:
        if (aLabel in rowLabelsA):
            numBinA += 1
        else:
            featLabelsBnotA += [aLabel]

    print " "
    print "     --> total # of features in each : %6d  %6d " % (len(rowLabelsA), len(rowLabelsB))
    print "         overlap counts              : %6d  %6d " % (numAinB, numBinA)

    featLabelsAnotB.sort()
    featLabelsBnotA.sort()

    ## print len(featLabelsAnotB), featLabelsAnotB[:5]
    ## print len(featLabelsBnotA), featLabelsBnotA[:5]

    print " "
    if (len(featLabelsAnotB) > 0):
        print " %d labels in A but not in B (string names have been arbitrarily forced to lower-case) : " % (len(featLabelsAnotB))
        for aLabel in featLabelsAnotB:
            print "     feat in A not B: ", aLabel
    else:
        print " NO labels in A but not in B (string names have been arbitrarily forced to lower-case) : "

    print " "
    if (len(featLabelsBnotA) > 0):
        print " %d labels in B but not in A (string names have been arbitrarily forced to lower-case) : " % (len(featLabelsBnotA))
        for aLabel in featLabelsBnotA:
            print "     feat in B not A: ", aLabel
    else:
        print " NO labels in B but not in A (string names have been arbitrarily forced to lower-case) : "

    # and next at the column (sample) labels ...

    # first check the sample ID lengths ...
    colLabelA_len = len(dataA['colLabels'][0])
    colLabelB_len = len(dataB['colLabels'][0])
    minLen = min(colLabelA_len, colLabelB_len)
    if (minLen < colLabelA_len):
        print " NOTE: sample IDs in A will be abbreviated ... eg from <%s> to <%s> " \
            % (dataA['colLabels'][0], dataA['colLabels'][0][:minLen])
    if (minLen < colLabelB_len):
        print " NOTE: sample IDs in B will be abbreviated ... eg from <%s> to <%s> " \
            % (dataB['colLabels'][0], dataB['colLabels'][0][:minLen])

    colLabelsA = []
    for aLabel in dataA['colLabels']:
        colLabelsA += [aLabel[:minLen].upper()]
    colLabelsB = []
    for aLabel in dataB['colLabels']:
        colLabelsB += [aLabel[:minLen].upper()]

    numAinB = 0
    samplesAnotB = []
    for aLabel in colLabelsA:
        if (aLabel in colLabelsB):
            numAinB += 1
        else:
            samplesAnotB += [aLabel]

    numBinA = 0
    samplesBnotA = []
    for aLabel in colLabelsB:
        if (aLabel in colLabelsA):
            numBinA += 1
        else:
            samplesBnotA += [aLabel]

    print " "
    print "     --> total # of samples in each  : %6d  %6d " % (len(colLabelsA), len(colLabelsB))
    print "         overlap counts              : %6d  %6d " % (numAinB, numBinA)

    samplesAnotB.sort()
    samplesBnotA.sort()

    print " "
    if (len(samplesAnotB) > 0):
        print " %d samples in A but not in B (string names have been arbitrarily forced to upper-case) : " % (len(samplesAnotB))
        for aLabel in samplesAnotB:
            print aLabel
    else:
        print " NO samples in A but not in B (string names have been arbitrarily forced to upper-case) : "

    print " "
    if (len(samplesBnotA) > 0):
        print " %d samples in B but not in A (string names have been arbitrarily forced to upper-case) : " % (len(samplesBnotA))
        for aLabel in samplesBnotA:
            print aLabel
    else:
        print " NO samples in B but not in A (string names have been arbitrarily forced to upper-case) : "

    # now, for the features that exist in both, compare the *data* values ...
    print " "
    print " "
    print " "
    print " now checking data values, feature by feature ... "
    print " (based on features in A that are also in B) "

    ## print rowLabelsA[:5]
    ## print featLabelsAnotB[:5]

    for aLabel in rowLabelsA:

        # skip this feature if it is in the A-not-B list ...
        if (aLabel in featLabelsAnotB):
            continue

        # skip binary indicator features since they are generated automatically
        # ...
        if (aLabel[6:9] == ":i("):
            continue

        # get the feature index for each matrix ...
        iA = findIndex(dataA['rowLabels'], aLabel)
        iB = findIndex(dataB['rowLabels'], aLabel)
        ## print " (a) ", aLabel, iA, iB

        # grab the data vector from each matrix ...
        dataVecA = dataA['dataMatrix'][iA]
        dataVecB = dataB['dataMatrix'][iB]
        # print dataVecA, dataVecB

        numDiff = 0
        numNotNA = 0

        sumDiff = 0.
        sumDiff2 = 0.
        sumN = 0

        # now loop over the individual data values for this feature
        ## print " looping jA over %d " % len(dataVecA)
        for jA in range(len(dataVecA)):
            sampleA = dataA['colLabels'][jA]
            ## print jA, sampleA, dataVecA[jA]

            if (str(dataVecA[jA]) == "NA" or str(dataVecA[jA]) == "-999999"):
                ## print " (NA) "
                continue

            else:
                numNotNA += 1

                jB = findIndex(dataB['colLabels'], sampleA)
                if (jB < 0):
                    ## print " sample <%s> not found in matrix B " % sampleA
                    if (str(dataVecA[jA]) == "NA"):
                        doNothing = 1
                    elif (str(dataVecA[jA]) == "-999999"):
                        doNothing = 1
                    else:
                        numDiff += 1
                else:
                    ## print jA, jB
                    ## print dataVecA[jA], dataVecB[jB]

                    if (str(dataVecB[jB]) == "NA" or str(dataVecB[jB]) == "-999999"):
                        doNothing = 1

                    elif (str(dataVecA[jA]).lower() != str(dataVecB[jB]).lower()):
                        # print " values are different for sample <%s> : <%s>
                        # vs <%s> " % ( sampleA, str(dataVecA[jA]),
                        # str(dataVecB[jB]) )
                        numDiff += 1
                        try:
                            diffVal = float (dataVecA[jA] ) - \
                                float(dataVecB[jB])
                            if (abs(diffVal) > 99999):
                                print " ERROR ??? using NA value ??? !!! ", jA, dataVecA[jA], jB, dataVecB[jB]
                            sumDiff += diffVal
                            sumDiff2 += (diffVal * diffVal)
                            sumN += 1
                        except:
                            doNothing = 1

        if (numDiff == 0):
            print " all %4d non-NA values are the same <%s> (%d) " % (numNotNA, aLabel, len(dataVecA))
        else:
            if (sumN > 3):
                meanDiff = float(sumDiff) / float(sumN)
                meanDiff2 = float(sumDiff2) / float(sumN)
                sigmDiff = math.sqrt(
                    max((meanDiff2 - (meanDiff * meanDiff)), 0.))
                if (abs(meanDiff) > 0.02 or sigmDiff > 0.02):
                    print " %4d out of %4d non-NA values are different <%s> (%d) [%d,%.2f,%.2f] " % \
                        (numDiff, numNotNA, aLabel,
                         len(dataVecA), sumN, meanDiff, sigmDiff)
                else:
                    print " %4d out of %4d non-NA values are different <%s> (%d) -- but not significantly [%d,%.2f,%.2f] " % \
                        (numDiff, numNotNA, aLabel,
                         len(dataVecA), sumN, meanDiff, sigmDiff)
            else:
                print " %4d out of %4d non-NA values are different <%s> (%d) " % (numDiff, numNotNA, aLabel, len(dataVecA))

    print " "
    print " FINISHED "
    print " "

# -#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#
<|endoftext|>