<commit_before># This file is part of victims-web.
#
# Copyright (C) 2013 The Victims Project
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""
Submission module. Handle submission related logic.
"""
import json

from os import makedirs, remove
from os.path import isdir, isfile, join
from uuid import uuid4
from subprocess import check_output, CalledProcessError

from flask import current_app
from werkzeug import secure_filename

from victims_web.models import Hash, Submission
from victims_web.plugin.charon import download

def groups():
    """
    Retrieve a list of groups with the default '---' group added.
    """
    submission_groups = {'---': []}
    if 'SUBMISSION_GROUPS' in current_app.config:
        configured_groups = current_app.config['SUBMISSION_GROUPS']
        for group in configured_groups:
            submission_groups[group] = configured_groups[group]
    return submission_groups

def allowed_groups():
    """
    Retrieve a list of groups that we know of. All configured group names are
    returned.
    """
    if 'SUBMISSION_GROUPS' in current_app.config:
        return current_app.config['SUBMISSION_GROUPS'].keys()
    return []

def group_keys(group):
    """
    Retrieve the metadata keys associated with a given group.
    """
    return groups().get(group, [])

def process_metadata(group, values={}, noprefix=False):
    """
    Process any group specific metadata that was provided in the submission
    form.
    """
    meta = {}
    current_groups = groups()
    if group.strip().lower() in current_groups:
        for field in current_groups[group]:
            if noprefix:
                name = field
            else:
                name = '%s-%s' % (group, field)
            if name in values:
                value = values[name].strip()
                if len(value) > 0:
                    meta[field] = value
    return meta

def get_hash(submission):
    """
    Helper method to process an archive at source where possible from a
    submission.
    """
    if not isfile(submission.source):
        return

    config = current_app.config
    key = 'HASHING_COMMANDS'

    if key not in config or submission.group not in config[key]:
        return

    command = config[key][submission.group].format(archive=submission.source)
    try:
        output = check_output(command, shell=True).strip()
        json_data = json.loads(output)
        json_data['cves'] = submission.cves
        entry = Hash()
        entry.load_json(submission.submitter, json_data)
        submission.entry = entry
        submission.approval = 'PENDING_APPROVAL'
        submission.validate()
        submission.save()

        # we are done safely, now remove the source
        try:
            remove(submission.source)
        except:
            current_app.logger.warn(
                'Deletion failed for %s' % (submission.source))
    except CalledProcessError:
        current_app.logger.debug(
            'Command execution failed for "%s"' % (command))
    except Exception as e:
        current_app.logger.warn('Failed to hash: ' + e.message)

def submit(submitter, source, group=None, filename=None, suffix=None, cves=[],
           meta={}, entry=None, approval='REQUESTED'):
    current_app.logger.info('Submitting: %s' % (
        ', '.join(['%s:%s' % (k, v) for (k, v) in locals().items()])))
    submission = Submission()
    submission.source = source
    submission.group = group
    submission.filename = filename
    if suffix:
        submission.format = suffix.title()
    submission.cves = cves
    submission.metadata = meta
    submission.submitter = submitter
    submission.entry = entry
    submission.approval = approval
    submission.validate()
    submission.save()

    if entry is None:
        # TODO: Make this async
        get_hash(submission)

    current_app.config['INDEX_REFRESH_FLAG'] = True

def get_upload_folder():
    """
    Helper methed to fetch configured upload directory. If the directory does
    not exist, it is created.
    """
    upload_dir = current_app.config['UPLOAD_FOLDER']
    if not isdir(upload_dir):
        current_app.logger.info('Creating upload directory: %s' % (upload_dir))
        makedirs(upload_dir, 0755)
    return upload_dir

def upload_file(archive):
    """
    Given a FileStorage object, the file is securely uploaded to the server
    to the configured upload directory. A random prefix is added to the
    filename.
    """
    if len(archive.filename) == 0:
        raise ValueError('No archive provided')

    upload_dir = get_upload_folder()

    suffix = archive.filename[archive.filename.rindex('.') + 1:]
    if suffix not in current_app.config['ALLOWED_EXTENSIONS']:
        raise ValueError('Invalid archive: %s' % (archive.filename))

    filename = secure_filename(archive.filename)
    sfilename = '%s-%s' % (str(uuid4()), filename)
    ondisk = join(upload_dir, sfilename)
    archive.save(ondisk)

    current_app.logger.info(
        'Uploaded %s' % (filename))

    return (ondisk, filename, suffix)

def upload_from_metadata(group, meta):
    """
    Given only metadata of an archive ask charon to retrive it where possible
    """
    if group not in allowed_groups():
        raise ValueError('Invalid group')
    return download(group, meta)

def upload(group, archive=None, meta=None):
    """
    Helper method to upload files using archive file in request or metadata
    provided. If no files get uploaded a ValueError is raised.
    """
    files = []

    try:
        if archive:
            files.append(upload_file(archive))
        else:
            raise ValueError('No Archive provided')
    except ValueError as ve:
        if meta:
            print(meta)
            files = upload_from_metadata(group, meta)
        else:
            raise ve

    if len(files) == 0:
        raise ValueError('Invalid submissions, no archives could be resolved.')

    return files
<commit_msg>Rename get_hash to set_hash<commit_after># This file is part of victims-web.
#
# Copyright (C) 2013 The Victims Project
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""
Submission module. Handle submission related logic.
"""
import json

from os import makedirs, remove
from os.path import isdir, isfile, join
from uuid import uuid4
from subprocess import check_output, CalledProcessError

from flask import current_app
from werkzeug import secure_filename

from victims_web.models import Hash, Submission
from victims_web.plugin.charon import download

def groups():
    """
    Retrieve a list of groups with the default '---' group added.
    """
    submission_groups = {'---': []}
    if 'SUBMISSION_GROUPS' in current_app.config:
        configured_groups = current_app.config['SUBMISSION_GROUPS']
        for group in configured_groups:
            submission_groups[group] = configured_groups[group]
    return submission_groups

def allowed_groups():
    """
    Retrieve a list of groups that we know of. All configured group names are
    returned.
    """
    if 'SUBMISSION_GROUPS' in current_app.config:
        return current_app.config['SUBMISSION_GROUPS'].keys()
    return []

def group_keys(group):
    """
    Retrieve the metadata keys associated with a given group.
    """
    return groups().get(group, [])

def process_metadata(group, values={}, noprefix=False):
    """
    Process any group specific metadata that was provided in the submission
    form.
    """
    meta = {}
    current_groups = groups()
    if group.strip().lower() in current_groups:
        for field in current_groups[group]:
            if noprefix:
                name = field
            else:
                name = '%s-%s' % (group, field)
            if name in values:
                value = values[name].strip()
                if len(value) > 0:
                    meta[field] = value
    return meta

def set_hash(submission):
    """
    Helper method to process an archive at source where possible from a
    submission.
    """
    if not isfile(submission.source):
        return

    config = current_app.config
    key = 'HASHING_COMMANDS'

    if key not in config or submission.group not in config[key]:
        return

    command = config[key][submission.group].format(archive=submission.source)
    try:
        output = check_output(command, shell=True).strip()
        json_data = json.loads(output)
        json_data['cves'] = submission.cves
        entry = Hash()
        entry.load_json(submission.submitter, json_data)
        submission.entry = entry
        submission.approval = 'PENDING_APPROVAL'
        submission.validate()
        submission.save()

        # we are done safely, now remove the source
        try:
            remove(submission.source)
        except:
            current_app.logger.warn(
                'Deletion failed for %s' % (submission.source))
    except CalledProcessError:
        current_app.logger.debug(
            'Command execution failed for "%s"' % (command))
    except Exception as e:
        current_app.logger.warn('Failed to hash: ' + e.message)

def submit(submitter, source, group=None, filename=None, suffix=None, cves=[],
           meta={}, entry=None, approval='REQUESTED'):
    current_app.logger.info('Submitting: %s' % (
        ', '.join(['%s:%s' % (k, v) for (k, v) in locals().items()])))
    submission = Submission()
    submission.source = source
    submission.group = group
    submission.filename = filename
    if suffix:
        submission.format = suffix.title()
    submission.cves = cves
    submission.metadata = meta
    submission.submitter = submitter
    submission.entry = entry
    submission.approval = approval
    submission.validate()
    submission.save()

    if entry is None:
        # TODO: Make this async
        set_hash(submission)

    current_app.config['INDEX_REFRESH_FLAG'] = True

def get_upload_folder():
    """
    Helper methed to fetch configured upload directory. If the directory does
    not exist, it is created.
    """
    upload_dir = current_app.config['UPLOAD_FOLDER']
    if not isdir(upload_dir):
        current_app.logger.info('Creating upload directory: %s' % (upload_dir))
        makedirs(upload_dir, 0755)
    return upload_dir

def upload_file(archive):
    """
    Given a FileStorage object, the file is securely uploaded to the server
    to the configured upload directory. A random prefix is added to the
    filename.
    """
    if len(archive.filename) == 0:
        raise ValueError('No archive provided')

    upload_dir = get_upload_folder()

    suffix = archive.filename[archive.filename.rindex('.') + 1:]
    if suffix not in current_app.config['ALLOWED_EXTENSIONS']:
        raise ValueError('Invalid archive: %s' % (archive.filename))

    filename = secure_filename(archive.filename)
    sfilename = '%s-%s' % (str(uuid4()), filename)
    ondisk = join(upload_dir, sfilename)
    archive.save(ondisk)

    current_app.logger.info(
        'Uploaded %s' % (filename))

    return (ondisk, filename, suffix)

def upload_from_metadata(group, meta):
    """
    Given only metadata of an archive ask charon to retrive it where possible
    """
    if group not in allowed_groups():
        raise ValueError('Invalid group')
    return download(group, meta)

def upload(group, archive=None, meta=None):
    """
    Helper method to upload files using archive file in request or metadata
    provided. If no files get uploaded a ValueError is raised.
    """
    files = []

    try:
        if archive:
            files.append(upload_file(archive))
        else:
            raise ValueError('No Archive provided')
    except ValueError as ve:
        if meta:
            print(meta)
            files = upload_from_metadata(group, meta)
        else:
            raise ve

    if len(files) == 0:
        raise ValueError('Invalid submissions, no archives could be resolved.')

    return files
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
# encoding: utf-8
"""
survey/core.py

Created by Karl Dubost on 2012-07-03.
Copyright (c) 2011 Grange. All rights reserved.
see LICENSE
"""

import urllib2
import urlparse
import requests
from lxml import etree
import cssutils
import os.path
import logging
from datetime import datetime

# CONSTANT - ALL OF THESE will have to be passed as arguements.
# List of URIs, 1 URI by line
SITELIST = os.path.dirname(__file__) + "/../tests/urlist-top1000.data"
# SITELIST = os.path.dirname(__file__) + "/../tests/urlist.data"
# UAREF = "Opera/9.80 (Macintosh; Intel Mac OS X 10.8.0) Presto/2.12.363 Version/12.50"
UAREF = "Opera/9.80 (Android 2.3.5; Linux; Opera Mobi/ADR-1202082305; U; en) Presto/2.10.254 Version/12.00"
UATEST = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6"
logging.basicConfig(filename='log_filename.txt', level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
VENDORPREFIX = ['moz', 'ms', 'o', 'webkit']

class UriCheck:
    """Some control about URIs from the file"""

    def __init__(self):
        self.uri = ""

    def ishttpURI(self, uri):
        """Given a URI,
        returns True if http or https
        returns False if not http or https"""
        req = urllib2.Request(uri)
        if req.get_type() in ['http', 'https']:
            return True
        else:
            return False

class HttpRequests:
    """Servers behave depending on what is asking which URI"""

    def __init__(self):
        self.content = ""
        self.statuscode = ""

    def getRequest(self, uri, uastring):
        """Given a URI and a UA String,
        returns a list with uri, newlocation, statuscode, content"""
        headers = {'User-Agent': uastring}
        r = requests.get(uri, headers=headers)
        statuscode = r.status_code
        responseheaders = r.headers
        history = r.history
        return statuscode, responseheaders, history

    def getContent(self, uri, useragentstring):
        """Return the content associated with an uri"""
        headers = {'User-Agent': useragentstring}
        r = requests.get(uri, headers=headers)
        # r.history contains a list of the different redirections
        # if needed later.
        finaluri = r.url
        responsetext = r.text
        # first test if it's a string
        try:
            isinstance(responsetext, basestring)
        # Log type errors
        except TypeError as e:
            logging.debug("HTMLTEXT - %s" % (e.message))
        else:
            # then test if it's unicode and convert it
            if isinstance(responsetext, unicode):
                htmltext = responsetext.encode('utf-8')
            # or if not it is just the common text
            elif isinstance(responsetext, str):
                htmltext = responsetext
        return htmltext, finaluri

class Css:
    """Grabing All CSS for one given URI"""

    def __init__(self):
        self.htmltext = ""

    def getCssUriList(self, htmltext, uri):
        """Given an htmltext, get the list of linked CSS"""
        cssurilist = []
        tree = etree.HTML(htmltext)
        csspathlist = tree.xpath('//link[@rel="stylesheet"]/@href')
        for i, csspath in enumerate(csspathlist):
            cssurl = urlparse.urljoin(uri, csspath)
            cssurilist.append(cssurl)
        return cssurilist

    def getCssHttpUriList(self, httpheadersdict, uri):
        """Given HTTP headers dictionary for a URI, extract the list of linked CSS"""
        # parse HTTP header for Link: <uri>;rel=stylesheet
        # requests returns something such as:
        # 'link': '</foo.css>;rel=stylesheet, </>;rel=next'
        # httplib might return a list of csv for a header.
        cssurilist = []
        req = HttpRequests()
        linkhttp = httpheadersdict['link']
        csspathlist = req.parseHttpLink(linkhttp)
        for i, csspath in enumerate(csspathlist):
            cssurl = urlparse.urljoin(uri, csspath)
            cssurilist.append(cssurl)
        return cssurilist

    def getCssRules(self, uri):
        """Given the URI of a CSS file,
        return the list of all CSS rules,
        including all the imports."""
        IMPORT_FLAG = False
        stylesheet = ""
        # requests is more robust for encoding errors
        headers = {'User-Agent': UAREF}
        r = requests.get(uri, headers=headers)
        stylesheet = cssutils.parseString(r.text)
        for cssrule in stylesheet:
            if cssrule.type == 3:
                IMPORT_FLAG = True
        # ResolveImports only if there is a rule
        if IMPORT_FLAG:
            stylesheet2 = cssutils.resolveImports(stylesheet)
        else:
            stylesheet2 = stylesheet
        return stylesheet2

    def hasStyleElement(self, htmltext):
        """Given an htmltext,
        check if the style element is available
        return True if yes"""
        tree = etree.HTML(htmltext)
        styleelements = tree.xpath('//style')
        if not styleelements:
            return False
        else:
            return True

    def getStyleElementRules(self, htmltext):
        """Given an htmltext,
        return the CSS rules contained in the content"""
        compiledstyle = ""
        stylesheet = cssutils.css.CSSStyleSheet()
        tree = etree.HTML(htmltext)
        styleelements = tree.xpath('//style')
        for styleelt in styleelements:
            if styleelt.text != None:
                compiledstyle = compiledstyle + styleelt.text
            else:
                logging.debug("STYLE ELEMENT %s on %s" % (styleelements.index(styleelt) + 1, len(styleelements)))
        if compiledstyle != None:
            cssutils.ser.prefs.indentClosingBrace = False
            cssutils.ser.prefs.keepComments = False
            cssutils.ser.prefs.lineSeparator = u''
            cssutils.ser.prefs.omitLastSemicolon = False
            try:
                stylesheet = cssutils.parseString(compiledstyle)
            except ValueError as e:
                logging.info("BOGUS STYLE RULE: %s" % (e.message))
        else:
            raise ValueError("STYLE ELEMENT: no CSS Rules")
        return stylesheet

    def hasVendorProperty(self, vendorname, propertyname, declarationslist):
        """Given a declarationslist, a vendor name, and propertyname,
        verify that the the property name for this vendor name exists
        return True"""
        expectedproperty = "-%s-%s" % (vendorname, propertyname)
        propertynamelist = declarationslist.keys()
        try:
            propertynamelist.index(expectedproperty)
            return True
        except ValueError:
            return False

    def hasProperty(self, propertyname, declarationslist):
        """Given a declarationslist, a propertyname,
        verify that the the property name exists
        return True"""
        propertynamelist = declarationslist.keys()
        try:
            propertynamelist.index(propertyname)
            return True
        except ValueError:
            return False

class Survey:
    """Mother of survey modules"""

    def __init__(self):
        # TODO: Initializing the reporting output mode
        pass

    def compareCssProperties(self, vendorref, vendorcomp, propertyname, cssrule):
        """compares how CSS properties are used and computes a score
        vendorref  = +1
        vendorcomp = +2
        prefixless = +4

        Example:
        .sel {  -o-color: #fff;
              -moz-color: #fff;
                   color: #fff;}

        will return a score of 5
        with property = 'color', vendorref = 'o', vendorcomp = 'moz'
        """
        score = 0
        propertyvendorlist = []
        propertyvendorlist.append(propertyname)
        css = Css()
        if css.hasVendorProperty(vendorref, propertyname, cssrule):
            score += 1
            propertyvendorlist.append(vendorref)
        if css.hasVendorProperty(vendorcomp, propertyname, cssrule):
            score += 2
            propertyvendorlist.append(vendorcomp)
        if css.hasProperty(propertyname, cssrule):
            score += 4
            propertyvendorlist.append('prefixless')
        return score, propertyvendorlist

    def getVendorPrefixes(self, propertyname, cssrule):
        """It will return the list of vendor prefixes for this propertyname"""
        propertyvendorlist = []
        propertyvendorlist.append(propertyname)
        css = Css()
        for vendorref in VENDORPREFIX:
            if css.hasVendorProperty(vendorref, propertyname, cssrule):
                propertyvendorlist.append(vendorref)
        if css.hasProperty(propertyname, cssrule):
            propertyvendorlist.append('prefixless')
        return propertyvendorlist

def main():
    cssutils.log.setLevel(logging.FATAL)
    uc = UriCheck()
    req = HttpRequests()
    css = Css()
    survey = Survey()
    propertytocheck = ['transition', 'transform', 'animation']
    with open(SITELIST) as f:
        logging.info("START - %s with %s" % (datetime.today().isoformat(), UAREF))
        for uri in f:
            uri = uri.strip()
           # remove leading, trailing spaces
            if uri.startswith("#"):
                continue
            elif uri == '':
                continue
            logging.info("SITE: %s" % (uri))
            if uc.ishttpURI(uri):
                try:
                    htmltext, finaluri = req.getContent(uri, UAREF)
                    if css.hasStyleElement(htmltext):
                        styleeltrule = css.getStyleElementRules(htmltext)
                        if styleeltrule != "":
                            logging.info("Style Element at %s" % (finaluri))
                            for cssrule in styleeltrule:
                                if cssrule.type == 1:
                                    for i, propertyname in enumerate(propertytocheck):
                                        score, propertyresultlist = survey.compareCssProperties('o', 'webkit', propertyname, cssrule.style)
                                        # printing only if the propertyname is found and if there is more than prefixless
                                        if score > 0 and score != 4:
                                            print propertyresultlist, finaluri

                    cssurislist = css.getCssUriList(htmltext, finaluri)
                    if len(cssurislist) == 0:
                        logging.info("NO CSS LINK REL at %s" % (finaluri))
                    else:
                        logging.info("CSS LINK REL: %s at %s" % (len(cssurislist), finaluri))
                    for cssuri in cssurislist:
                        cssruleslist = css.getCssRules(cssuri)
                        try:
                            cssruleslist = css.getCssRules(cssuri)
                        except ValueError as e:
                            logging.info("CSS Mime Type error: %s" % (e.message))
                        for cssrule in cssruleslist:
                            if cssrule.type == 1:
                                # This is a rule, we process
                                # Reminder cssrule.type == 0 -> comment
                                for i, propertyname in enumerate(propertytocheck):
                                    score, propertyresultlist = survey.compareCssProperties('o', 'webkit', propertyname, cssrule.style)
                                    propertyfulllist = survey.getVendorPrefixes(propertyname, cssrule.style)
                                    # printing only if the propertyname is found and if there is more than prefixless
                                    # if score > 0 and score != 4:
                                    if score > 0:
                                        # print finaluri, propertyresultlist
                                        print finaluri, propertyfulllist
                                        logging.info("SURVEY: %s %s" % (finaluri, propertyresultlist))
                        # logging.info(responseheaders)

                except requests.exceptions.ConnectionError as e:
                    # we are catching network connection error and record them.
                    logging.info("Connection error: %s" % (e.message))

if __name__ == '__main__':
    main()
<commit_msg>when parsing fails for getContent logged fonction where it failed<commit_after>#!/usr/bin/env python2.7
# encoding: utf-8
"""
survey/core.py

Created by Karl Dubost on 2012-07-03.
Copyright (c) 2011 Grange. All rights reserved.
see LICENSE
"""

import urllib2
import urlparse
import requests
from lxml import etree
import cssutils
import os.path
import logging
from datetime import datetime

# CONSTANT - ALL OF THESE will have to be passed as arguements.
# List of URIs, 1 URI by line
SITELIST = os.path.dirname(__file__) + "/../tests/urlist-top1000.data"
# SITELIST = os.path.dirname(__file__) + "/../tests/urlist.data"
# UAREF = "Opera/9.80 (Macintosh; Intel Mac OS X 10.8.0) Presto/2.12.363 Version/12.50"
UAREF = "Opera/9.80 (Android 2.3.5; Linux; Opera Mobi/ADR-1202082305; U; en) Presto/2.10.254 Version/12.00"
UATEST = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6"
logging.basicConfig(filename='log_filename.txt', level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
VENDORPREFIX = ['moz', 'ms', 'o', 'webkit']

class UriCheck:
    """Some control about URIs from the file"""

    def __init__(self):
        self.uri = ""

    def ishttpURI(self, uri):
        """Given a URI,
        returns True if http or https
        returns False if not http or https"""
        req = urllib2.Request(uri)
        if req.get_type() in ['http', 'https']:
            return True
        else:
            return False

class HttpRequests:
    """Servers behave depending on what is asking which URI"""

    def __init__(self):
        self.content = ""
        self.statuscode = ""

    def getRequest(self, uri, uastring):
        """Given a URI and a UA String,
        returns a list with uri, newlocation, statuscode, content"""
        headers = {'User-Agent': uastring}
        r = requests.get(uri, headers=headers)
        statuscode = r.status_code
        responseheaders = r.headers
        history = r.history
        return statuscode, responseheaders, history

    def getContent(self, uri, useragentstring):
        """Return the content associated with an uri"""
        headers = {'User-Agent': useragentstring}
        r = requests.get(uri, headers=headers)
        # r.history contains a list of the different redirections
        # if needed later.
        finaluri = r.url
        responsetext = r.text
        # first test if it's a string
        try:
            isinstance(responsetext, basestring)
        # Log type errors
        except TypeError as e:
            logging.debug("getContent - %s - %s" % (uri, e.message))
        else:
            # then test if it's unicode and convert it
            if isinstance(responsetext, unicode):
                htmltext = responsetext.encode('utf-8')
            # or if not it is just the common text
            elif isinstance(responsetext, str):
                htmltext = responsetext
        return htmltext, finaluri

class Css:
    """Grabing All CSS for one given URI"""

    def __init__(self):
        self.htmltext = ""

    def getCssUriList(self, htmltext, uri):
        """Given an htmltext, get the list of linked CSS"""
        cssurilist = []
        tree = etree.HTML(htmltext)
        csspathlist = tree.xpath('//link[@rel="stylesheet"]/@href')
        for i, csspath in enumerate(csspathlist):
            cssurl = urlparse.urljoin(uri, csspath)
            cssurilist.append(cssurl)
        return cssurilist

    def getCssHttpUriList(self, httpheadersdict, uri):
        """Given HTTP headers dictionary for a URI, extract the list of linked CSS"""
        # parse HTTP header for Link: <uri>;rel=stylesheet
        # requests returns something such as:
        # 'link': '</foo.css>;rel=stylesheet, </>;rel=next'
        # httplib might return a list of csv for a header.
        cssurilist = []
        req = HttpRequests()
        linkhttp = httpheadersdict['link']
        csspathlist = req.parseHttpLink(linkhttp)
        for i, csspath in enumerate(csspathlist):
            cssurl = urlparse.urljoin(uri, csspath)
            cssurilist.append(cssurl)
        return cssurilist

    def getCssRules(self, uri):
        """Given the URI of a CSS file,
        return the list of all CSS rules,
        including all the imports."""
        IMPORT_FLAG = False
        stylesheet = ""
        # requests is more robust for encoding errors
        headers = {'User-Agent': UAREF}
        r = requests.get(uri, headers=headers)
        stylesheet = cssutils.parseString(r.text)
        for cssrule in stylesheet:
            if cssrule.type == 3:
                IMPORT_FLAG = True
        # ResolveImports only if there is a rule
        if IMPORT_FLAG:
            stylesheet2 = cssutils.resolveImports(stylesheet)
        else:
            stylesheet2 = stylesheet
        return stylesheet2

    def hasStyleElement(self, htmltext):
        """Given an htmltext,
        check if the style element is available
        return True if yes"""
        tree = etree.HTML(htmltext)
        styleelements = tree.xpath('//style')
        if not styleelements:
            return False
        else:
            return True

    def getStyleElementRules(self, htmltext):
        """Given an htmltext,
        return the CSS rules contained in the content"""
        compiledstyle = ""
        stylesheet = cssutils.css.CSSStyleSheet()
        tree = etree.HTML(htmltext)
        styleelements = tree.xpath('//style')
        for styleelt in styleelements:
            if styleelt.text != None:
                compiledstyle = compiledstyle + styleelt.text
            else:
                logging.debug("STYLE ELEMENT %s on %s" % (styleelements.index(styleelt) + 1, len(styleelements)))
        if compiledstyle != None:
            cssutils.ser.prefs.indentClosingBrace = False
            cssutils.ser.prefs.keepComments = False
            cssutils.ser.prefs.lineSeparator = u''
            cssutils.ser.prefs.omitLastSemicolon = False
            try:
                stylesheet = cssutils.parseString(compiledstyle)
            except ValueError as e:
                logging.info("BOGUS STYLE RULE: %s" % (e.message))
        else:
            raise ValueError("STYLE ELEMENT: no CSS Rules")
        return stylesheet

    def hasVendorProperty(self, vendorname, propertyname, declarationslist):
        """Given a declarationslist, a vendor name, and propertyname,
        verify that the the property name for this vendor name exists
        return True"""
        expectedproperty = "-%s-%s" % (vendorname, propertyname)
        propertynamelist = declarationslist.keys()
        try:
            propertynamelist.index(expectedproperty)
            return True
        except ValueError:
            return False

    def hasProperty(self, propertyname, declarationslist):
        """Given a declarationslist, a propertyname,
        verify that the the property name exists
        return True"""
        propertynamelist = declarationslist.keys()
        try:
            propertynamelist.index(propertyname)
            return True
        except ValueError:
            return False

class Survey:
    """Mother of survey modules"""

    def __init__(self):
        # TODO: Initializing the reporting output mode
        pass

    def compareCssProperties(self, vendorref, vendorcomp, propertyname, cssrule):
        """compares how CSS properties are used and computes a score
        vendorref  = +1
        vendorcomp = +2
        prefixless = +4

        Example:
        .sel {  -o-color: #fff;
              -moz-color: #fff;
                   color: #fff;}

        will return a score of 5
        with property = 'color', vendorref = 'o', vendorcomp = 'moz'
        """
        score = 0
        propertyvendorlist = []
        propertyvendorlist.append(propertyname)
        css = Css()
        if css.hasVendorProperty(vendorref, propertyname, cssrule):
            score += 1
            propertyvendorlist.append(vendorref)
        if css.hasVendorProperty(vendorcomp, propertyname, cssrule):
            score += 2
            propertyvendorlist.append(vendorcomp)
        if css.hasProperty(propertyname, cssrule):
            score += 4
            propertyvendorlist.append('prefixless')
        return score, propertyvendorlist

    def getVendorPrefixes(self, propertyname, cssrule):
        """It will return the list of vendor prefixes for this propertyname"""
        propertyvendorlist = []
        propertyvendorlist.append(propertyname)
        css = Css()
        for vendorref in VENDORPREFIX:
            if css.hasVendorProperty(vendorref, propertyname, cssrule):
                propertyvendorlist.append(vendorref)
        if css.hasProperty(propertyname, cssrule):
            propertyvendorlist.append('prefixless')
        return propertyvendorlist

def main():
    cssutils.log.setLevel(logging.FATAL)
    uc = UriCheck()
    req = HttpRequests()
    css = Css()
    survey = Survey()
    propertytocheck = ['transition', 'transform', 'animation']
    with open(SITELIST) as f:
        logging.info("START - %s with %s" % (datetime.today().isoformat(), UAREF))
        for uri in f:
            uri = uri.strip()
           # remove leading, trailing spaces
            if uri.startswith("#"):
                continue
            elif uri == '':
                continue
            logging.info("SITE: %s" % (uri))
            if uc.ishttpURI(uri):
                try:
                    htmltext, finaluri = req.getContent(uri, UAREF)
                    if css.hasStyleElement(htmltext):
                        styleeltrule = css.getStyleElementRules(htmltext)
                        if styleeltrule != "":
                            logging.info("Style Element at %s" % (finaluri))
                            for cssrule in styleeltrule:
                                if cssrule.type == 1:
                                    for i, propertyname in enumerate(propertytocheck):
                                        score, propertyresultlist = survey.compareCssProperties('o', 'webkit', propertyname, cssrule.style)
                                        # printing only if the propertyname is found and if there is more than prefixless
                                        if score > 0 and score != 4:
                                            print propertyresultlist, finaluri

                    cssurislist = css.getCssUriList(htmltext, finaluri)
                    if len(cssurislist) == 0:
                        logging.info("NO CSS LINK REL at %s" % (finaluri))
                    else:
                        logging.info("CSS LINK REL: %s at %s" % (len(cssurislist), finaluri))
                    for cssuri in cssurislist:
                        cssruleslist = css.getCssRules(cssuri)
                        try:
                            cssruleslist = css.getCssRules(cssuri)
                        except ValueError as e:
                            logging.info("CSS Mime Type error: %s" % (e.message))
                        for cssrule in cssruleslist:
                            if cssrule.type == 1:
                                # This is a rule, we process
                                # Reminder cssrule.type == 0 -> comment
                                for i, propertyname in enumerate(propertytocheck):
                                    score, propertyresultlist = survey.compareCssProperties('o', 'webkit', propertyname, cssrule.style)
                                    propertyfulllist = survey.getVendorPrefixes(propertyname, cssrule.style)
                                    # printing only if the propertyname is found and if there is more than prefixless
                                    # if score > 0 and score != 4:
                                    if score > 0:
                                        # print finaluri, propertyresultlist
                                        print finaluri, propertyfulllist
                                        logging.info("SURVEY: %s %s" % (finaluri, propertyresultlist))
                        # logging.info(responseheaders)

                except requests.exceptions.ConnectionError as e:
                    # we are catching network connection error and record them.
                    logging.info("Connection error: %s" % (e.message))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python3

import sys
import glob
import subprocess
import re
import collections
import os.path
import shlex
import yaml # pip3 install pyyaml
from timeit import default_timer as timer

OVERVIEW = False # with True Goblint isn't executed
GOBLINT_COMMAND = "./goblint --enable ana.sv-comp --disable ana.int.trier --enable ana.int.enums --enable ana.int.interval --sets solver td3 --enable exp.widen-context --enable exp.malloc-fail --enable exp.partition-arrays.enabled {code_filename}"
TIMEOUT = 30 # with some int that's Goblint timeout for single execution
START = 1

def extract_bool(p, s):
    m = re.search(p, s)
    return bool(m.group(1)) if m else None

stats = collections.defaultdict(int)
total_time = 0
try:
    inp = sys.argv[1]
    task_filenames = []
    if inp.endswith('.yml'):
        task_filenames.append(inp)
    elif inp.endswith('.set'):
        with open(inp) as set_file:
            for pattern in set_file:
                pattern = pattern.strip()
                if pattern:
                    pattern = os.path.join(os.path.dirname(inp), pattern)
                    for task_filename in glob.iglob(pattern):
                        task_filenames.append(task_filename)
    else:
        print('Call script on .yml or .set file') # TODO also allow .c to just call goblint on with above options?

    for task_i, task_filename in list(enumerate(sorted(task_filenames)))[START - 1:]:
        print(f"{task_i + 1}/{len(task_filenames)}: {task_filename}: ", end="", flush=True)

        if task_filename.endswith(".yml"):
            with open(task_filename) as task_file:
                y = yaml.safe_load(task_file)

                code_filename = y["input_files"]
                code_filename = os.path.join(os.path.dirname(task_filename), code_filename)

                expected = None
                for y_property in y["properties"]:
                    if y_property["property_file"] == "../properties/unreach-call.prp":
                        expected = y_property["expected_verdict"]
        else:
            code_filename = task_filename
            expected = extract_bool(r"_(false|true)-unreach-call", task_filename)

        if OVERVIEW:
            actual = None
            task_time = None
        else:
            start_time = timer()
            try:
                p = subprocess.run(shlex.split(GOBLINT_COMMAND.format(code_filename=code_filename)), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8", timeout=TIMEOUT)
                if "Fatal error: exception " in p.stdout:
                    print(p.stdout)
                    exit(1)
                actual = extract_bool(r"SV-COMP \(unreach-call\): (false|true)", p.stdout)
            except subprocess.TimeoutExpired:
                actual = "timeout"
            finally:
                end_time = timer()
                task_time = end_time - start_time
                total_time += task_time

        if "p" in locals(): # sometimes on timeout p is declared, sometimes isn't
            missing_funcs = False
            for m in re.finditer(r"Function definition missing for (__VERIFIER_\S+)", p.stdout):
                missing_funcs = True
                print(f"MISSING FUNC {m.group(1)}")

            if missing_funcs:
                sys.exit(1)

        text = None
        if expected is None or actual is None:
            text = f"NONE expected {expected}, actual {actual}"
        elif actual == expected:
            text = f"CORRECT {expected}"
        elif actual == "timeout":
            text = f"UNKNOWN expected {expected}, actual {actual}"
        else:
            text = f"INCORRECT expected {expected}, actual {actual}"

        time_text = f" ({task_time:.2f} s)" if task_time is not None else ""
        print(text + time_text)
        stats[text] += 1

except KeyboardInterrupt:
    pass
finally:
    print()
    print("-" * 80)
    print("For each .yml we compare with expected_verdict for 'c/properties/unreach-call.prp':")
    print("CHECK( init(main()), LTL(G ! call(__VERIFIER_error())) )")
    print("-> 'expected True' means for all paths __VERIFIER_error is never called (__VERIFIER_assert always true) ")
    print("-> 'expected False' means there exists a path where  __VERIFIER_error could be called (__VERIFIER_assert can be false)")
    print("-" * 80)
    for text, count in stats.items():
        print(f"{text}: {count}")
    print("-" * 80)
    print(f"total time: {total_time:.2f} s")
<commit_msg>Revert "str2bool is bool"<commit_after>#!/usr/bin/python3

import sys
import glob
import subprocess
import re
import collections
import os.path
import shlex
import yaml # pip3 install pyyaml
from timeit import default_timer as timer

OVERVIEW = False # with True Goblint isn't executed
GOBLINT_COMMAND = "./goblint --enable ana.sv-comp --disable ana.int.trier --enable ana.int.enums --enable ana.int.interval --sets solver td3 --enable exp.widen-context --enable exp.malloc-fail --enable exp.partition-arrays.enabled {code_filename}"
TIMEOUT = 30 # with some int that's Goblint timeout for single execution
START = 1

def str2bool(s):
    return {
        "true": True,
        "false": False
    }[s]

def extract_bool(p, s):
    m = re.search(p, s)
    return str2bool(m.group(1)) if m else None

stats = collections.defaultdict(int)
total_time = 0
try:
    inp = sys.argv[1]
    task_filenames = []
    if inp.endswith('.yml'):
        task_filenames.append(inp)
    elif inp.endswith('.set'):
        with open(inp) as set_file:
            for pattern in set_file:
                pattern = pattern.strip()
                if pattern:
                    pattern = os.path.join(os.path.dirname(inp), pattern)
                    for task_filename in glob.iglob(pattern):
                        task_filenames.append(task_filename)
    else:
        print('Call script on .yml or .set file') # TODO also allow .c to just call goblint on with above options?

    for task_i, task_filename in list(enumerate(sorted(task_filenames)))[START - 1:]:
        print(f"{task_i + 1}/{len(task_filenames)}: {task_filename}: ", end="", flush=True)

        if task_filename.endswith(".yml"):
            with open(task_filename) as task_file:
                y = yaml.safe_load(task_file)

                code_filename = y["input_files"]
                code_filename = os.path.join(os.path.dirname(task_filename), code_filename)

                expected = None
                for y_property in y["properties"]:
                    if y_property["property_file"] == "../properties/unreach-call.prp":
                        expected = y_property["expected_verdict"]
        else:
            code_filename = task_filename
            expected = extract_bool(r"_(false|true)-unreach-call", task_filename)

        if OVERVIEW:
            actual = None
            task_time = None
        else:
            start_time = timer()
            try:
                p = subprocess.run(shlex.split(GOBLINT_COMMAND.format(code_filename=code_filename)), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8", timeout=TIMEOUT)
                if "Fatal error: exception " in p.stdout:
                    print(p.stdout)
                    exit(1)
                actual = extract_bool(r"SV-COMP \(unreach-call\): (false|true)", p.stdout)
            except subprocess.TimeoutExpired:
                actual = "timeout"
            finally:
                end_time = timer()
                task_time = end_time - start_time
                total_time += task_time

        if "p" in locals(): # sometimes on timeout p is declared, sometimes isn't
            missing_funcs = False
            for m in re.finditer(r"Function definition missing for (__VERIFIER_\S+)", p.stdout):
                missing_funcs = True
                print(f"MISSING FUNC {m.group(1)}")

            if missing_funcs:
                sys.exit(1)

        text = None
        if expected is None or actual is None:
            text = f"NONE expected {expected}, actual {actual}"
        elif actual == expected:
            text = f"CORRECT {expected}"
        elif actual == "timeout":
            text = f"UNKNOWN expected {expected}, actual {actual}"
        else:
            text = f"INCORRECT expected {expected}, actual {actual}"

        time_text = f" ({task_time:.2f} s)" if task_time is not None else ""
        print(text + time_text)
        stats[text] += 1

except KeyboardInterrupt:
    pass
finally:
    print()
    print("-" * 80)
    print("For each .yml we compare with expected_verdict for 'c/properties/unreach-call.prp':")
    print("CHECK( init(main()), LTL(G ! call(__VERIFIER_error())) )")
    print("-> 'expected True' means for all paths __VERIFIER_error is never called (__VERIFIER_assert always true) ")
    print("-> 'expected False' means there exists a path where  __VERIFIER_error could be called (__VERIFIER_assert can be false)")
    print("-" * 80)
    for text, count in stats.items():
        print(f"{text}: {count}")
    print("-" * 80)
    print(f"total time: {total_time:.2f} s")
<|endoftext|>


<commit_before>#!/net/kodiak/volumes/lake/shared/users/marghoob/my_env/bin/python

import logging

logger = logging.getLogger(__name__)

import copy
import bisect
import pysam
import pybedtools
import vcf

svs_of_interest = ["DEL", "INS", "DUP", "DUP:TANDEM", "INV"]
sv_sources = ["Pindel", "BreakSeq", "HaplotypeCaller", "BreakDancer", "CNVnator"] # order is important!
precise_sv_sources = ["Pindel", "BreakSeq", "HaplotypeCaller"]
sv_sources_to_type = {"Pindel": "SR", "BreakSeq": "JM", "BreakDancer": "RP", "CNVnator": "RD", "HaplotypeCaller": "AS"}

class SVInterval:
    def __init__(self, chrom=None, start=0, end=0, name=None, sv_type=None, length=0, sources=set(), gt="./1", wiggle=0,
                 info=None, cipos=[], ciend=[], native_sv=None):
        self.chrom = chrom
        self.start = start
        self.end = end
        self.name = name
        self.sv_type = sv_type
        self.length = length
        self.info = info
        self.sources = sources
        self.gt = gt
        self.wiggle = wiggle
        self.sub_intervals = []
        self.is_precise = False
        self.is_validated = False
        self.validating_interval = None
        self.cipos = cipos
        self.ciend = ciend
        self.native_sv = native_sv

    def merge(self, interval):
        self.start = min(self.start, interval.start - interval.wiggle)
        self.end = max(self.end, interval.end + interval.wiggle)
        self.length = max(self.length, interval.length)
        self.name = self.name + "," + interval.name
        self.sub_intervals.append(interval)
        self.sources |= interval.sources

    def set_merged(self, interval1, interval2):
        self.chrom = interval1.chrom
        self.start = min(interval1.start - interval1.wiggle, interval2.start - interval2.wiggle)
        self.end = max(interval1.end + interval1.wiggle, interval2.end + interval2.wiggle)
        self.length = max(interval1.length, interval2.length)
        self.name = interval1.name + "," + interval2.name
        self.sv_type = interval1.sv_type
        self.info = None
        self.sub_intervals = [interval1, interval2]
        self.sources = interval1.sources | interval2.sources
        self.gt = interval1.gt

    def __lt__(self, other):
        if self.chrom != other.chrom: return self.chrom < other.chrom
        if self.start != other.start: return self.start < other.start
        return self.end < other.end

    def __str__(self):
        if self.sub_intervals:
            return ",".join([str(interval) for interval in self.sub_intervals])
        return "%s-%d-%d-%d-%s" % (self.chrom, self.start, self.end, self.length, ",".join(list(self.sources)))

    def __repr__(self):
        return "<" + self.__class__.__name__ + " " + str(self.__dict__) + ">"

    def overlaps(self, other, min_fraction_self=1e-9, min_fraction_other=1e-9, min_overlap_length_self=1,
                 min_overlap_length_other=1):
        if self.chrom != other.chrom: return False
        if max(self.start - self.wiggle, other.start - other.wiggle) >= min(self.end + self.wiggle,
                                                                            other.end + other.wiggle): return False

        self_length = float(self.end - self.start + 2 * self.wiggle)
        other_length = float(other.end - other.start + 2 * other.wiggle)
        overlap_length = min(self.end + self.wiggle, other.end + other.wiggle) - max(self.start - self.wiggle,
                                                                                other.start - other.wiggle)
        return float(overlap_length) >= max(min_fraction_self * self_length,
                                            min_fraction_other * other_length) and overlap_length >= max(
            min_fraction_self, min_overlap_length_other)

    def is_adjacent(self, other, gap=0):
        if self.chrom != other.chrom: return False
        return (self.end + gap >= other.start and self.end + gap < other.end) or (
        other.end + gap >= self.start and other.end + gap < self.end)

    def get_start(self):
        if not self.sub_intervals:
            return self.start

        return min([interval.get_start() for interval in self.sub_intervals])

    def get_end(self):
        if not self.sub_intervals:
            return self.end

        return max([interval.get_end() for interval in self.sub_intervals])

    def do_validation(self, overlap_ratio=0.5):
        self.start = self.get_start()
        self.end = self.get_end()

        if self.sv_type not in svs_of_interest:
            return

        if not self.sub_intervals:
            # This interval did not overlap anything
            self.is_precise = list(self.sources)[0] in precise_sv_sources
            # if self.sv_type == "INV" and self.length >= 1000: self.is_validated = True
            return

        lists = {source: [] for source in sv_sources}

        precise_merged = merge_intervals(
            [interval for interval in self.sub_intervals if list(interval.sources)[0] in precise_sv_sources])

        for interval in self.sub_intervals:
            # logger.debug("Examining subinterval " + str(interval))
            lists[list(interval.sources)[0]].append(interval)

        for source in sv_sources:
            # If the SV is called multiple times by the same tool, don't validate
            if len(lists[source]) == 1:
                if self.overlaps(lists[source][0], overlap_ratio, overlap_ratio) \
                        or interval_overlaps_interval_list(lists[source][0], precise_merged, overlap_ratio,
                                                           overlap_ratio):
                    self.is_validated = len(self.sources) > 1
                    self.validating_interval = lists[source][0]
                    self.is_precise = source in precise_sv_sources
                    break # This break makes the order of sv_sources important

        if not self.is_validated:
            return

        if self.is_precise and self.validating_interval is not None:
            self.start = self.validating_interval.start
            self.end = self.validating_interval.end
            self.length = self.validating_interval.length
            self.gt = self.validating_interval.gt
            self.info = self.validating_interval.info

            if self.length == 0:
                for source in sv_sources:
                    if source in self.sources:
                        if lists[source] and lists[source][0].length > 0:
                            self.length = lists[source][0].length
                            break

    def fix_pos(self):
        if self.sv_type == "INS":
            mid = (self.start + self.end) / 2
            if not self.is_precise:
                self.end = self.start
                self.cipos = ["0", "%d" % (self.end - self.start)]
                # self.cipos = ["%d" % (mid - self.start), "%d" % (mid - self.end)]
            else:
                self.start = mid
                self.end = mid

    def to_vcf_record(self, fasta_handle, sample):
        if self.start <= 0: return None
        if self.sv_type not in svs_of_interest: return None

        # ignore private haplotype caller calls
        if ((not self.sub_intervals) or len(self.sources) == 1) and list(self.sources)[0] == "HaplotypeCaller":
            return None

        # formulate the INFO field
        info = {}
        if not self.sub_intervals:
            info.update(self.info)
        else:
            for interval in self.sub_intervals:
                #TODO: this will just overwrite the other dict entries... this should be ok for pass variants
                info.update(interval.info)
        svmethods = [sv_sources_to_type[tool] for tool in self.sources]
        svmethods.sort()
        sv_len = -self.length if self.sv_type == "DEL" else self.length
        info = {"SVLEN": sv_len,
                "SVTYPE": self.sv_type,
                "SVMETHOD": ",".join(svmethods)}
        if self.sv_type == "DEL" or self.sv_type == "DUP":
            info["END"] = self.end
        else:
            return None

        if not self.is_precise:
            info.update({"IMPRECISE": None})

        info.update({"VT":"SV"})
        info.update({"SVTOOL":"MetaSVMerge"})
        info.update({"NUM_SVMETHODS":len(self.sources)})
        info.update({"SOURCES":str(self)})
        if self.cipos:
            info.update({"CIPOS": (",".join([str(a) for a in self.cipos]))})

        vcf_record = vcf.model._Record(self.chrom,
                                       self.start,
                                       ".",
                                       fasta_handle.fetch(self.chrom, self.start - 1, self.start),
                                       ["<%s>" % (self.sv_type)],
                                       ".",
                                       "PASS" if self.is_validated else "LowQual",
                                       info,
                                       "GT",
                                       [0],
                                       [vcf.model._Call(None, sample, vcf.model.make_calldata_tuple("GT")(GT="1/1"))])
        return vcf_record

    def to_bed_interval(self, sample_name):
        if self.start <= 0: return None
        if self.sv_type not in ["DEL", "INS", "INV"]: return None
        # if not self.sub_intervals and list(self.sources)[0] == "HaplotypeCaller": return None
        #if len(self.sources) == 1 and list(self.sources)[0] == "HaplotypeCaller": return None
        end = self.end if self.sv_type != "INS" else (self.end + 1)

        return pybedtools.Interval(self.chrom, self.start, end, name="%s,%d,%s" % (
        self.sv_type, self.length, ";".join(sorted([sv_sources_to_type[tool] for tool in self.sources]))),
                                   score=str(len(self.sources)))

    def to_svp_record(self, sample_name, id_num):
        if self.start <= 0: return None
        if self.sv_type not in ["DEL", "INS", "INV"]: return None
        if not self.sub_intervals and list(self.sources)[0] == "HaplotypeCaller": return None
        if len(self.sources) == 1 and list(self.sources)[0] == "HaplotypeCaller": return None

        start_outer = self.start
        start_inner = self.start
        end_inner = self.end
        end_outer = self.end
        type_of_computational_approach = ",".join(sorted([sv_sources_to_type[tool] for tool in self.sources]))

        return "%s\t%d\t%d\t%d\t%d\t%s\t%d\t%s\t%s\t%s\t%s\tMetaSV\t%d" % (
        self.chrom, start_outer, start_inner, end_inner, end_outer, self.sv_type, self.length, "BWA", "Illumina",
        sample_name, type_of_computational_approach, id_num)

def interval_overlaps_interval_list(interval, interval_list, min_fraction_self=1e-9, min_fraction_other=1e-9):
    index = bisect.bisect_left(interval_list, interval)
    if index > 0 and interval.overlaps(interval_list[index - 1], min_fraction_self, min_fraction_other,
                                       min_overlap_length_self=1, min_overlap_length_other=1):
        return True
    if index < len(interval_list) and interval.overlaps(interval_list[index], min_fraction_self, min_fraction_other,
                                                        min_overlap_length_self=1, min_overlap_length_other=1):
        return True
    return False

def merge_intervals(interval_list):
    if not interval_list: return []

    interval_list.sort()
    merged_intervals = []

    current_merged_interval = copy.deepcopy(interval_list[0])
    for i in xrange(len(interval_list) - 1):
        next_interval = interval_list[i + 1]
        if current_merged_interval.overlaps(next_interval) or current_merged_interval.is_adjacent(next_interval):
            if current_merged_interval.sub_intervals:
                current_merged_interval.merge(next_interval)
            else:
                new_merged_interval = SVInterval()
                # logger.debug("Merging %s with %s" % (repr(current_merged_interval), repr(next_interval)))
                new_merged_interval.set_merged(current_merged_interval, next_interval)
                current_merged_interval = new_merged_interval
        else:
            merged_intervals.append(current_merged_interval)
            current_merged_interval = copy.deepcopy(next_interval)
    merged_intervals.append(current_merged_interval)
    merged_intervals.sort()
    return merged_intervals
<commit_msg>don't do this...<commit_after>#!/net/kodiak/volumes/lake/shared/users/marghoob/my_env/bin/python

import logging

logger = logging.getLogger(__name__)

import copy
import bisect
import pysam
import pybedtools
import vcf

svs_of_interest = ["DEL", "INS", "DUP", "DUP:TANDEM", "INV"]
sv_sources = ["Pindel", "BreakSeq", "HaplotypeCaller", "BreakDancer", "CNVnator"] # order is important!
precise_sv_sources = ["Pindel", "BreakSeq", "HaplotypeCaller"]
sv_sources_to_type = {"Pindel": "SR", "BreakSeq": "JM", "BreakDancer": "RP", "CNVnator": "RD", "HaplotypeCaller": "AS"}

class SVInterval:
    def __init__(self, chrom=None, start=0, end=0, name=None, sv_type=None, length=0, sources=set(), gt="./1", wiggle=0,
                 info=None, cipos=[], ciend=[], native_sv=None):
        self.chrom = chrom
        self.start = start
        self.end = end
        self.name = name
        self.sv_type = sv_type
        self.length = length
        self.info = info
        self.sources = sources
        self.gt = gt
        self.wiggle = wiggle
        self.sub_intervals = []
        self.is_precise = False
        self.is_validated = False
        self.validating_interval = None
        self.cipos = cipos
        self.ciend = ciend
        self.native_sv = native_sv

    def merge(self, interval):
        self.start = min(self.start, interval.start - interval.wiggle)
        self.end = max(self.end, interval.end + interval.wiggle)
        self.length = max(self.length, interval.length)
        self.name = self.name + "," + interval.name
        self.sub_intervals.append(interval)
        self.sources |= interval.sources

    def set_merged(self, interval1, interval2):
        self.chrom = interval1.chrom
        self.start = min(interval1.start - interval1.wiggle, interval2.start - interval2.wiggle)
        self.end = max(interval1.end + interval1.wiggle, interval2.end + interval2.wiggle)
        self.length = max(interval1.length, interval2.length)
        self.name = interval1.name + "," + interval2.name
        self.sv_type = interval1.sv_type
        self.info = None
        self.sub_intervals = [interval1, interval2]
        self.sources = interval1.sources | interval2.sources
        self.gt = interval1.gt

    def __lt__(self, other):
        if self.chrom != other.chrom: return self.chrom < other.chrom
        if self.start != other.start: return self.start < other.start
        return self.end < other.end

    def __str__(self):
        if self.sub_intervals:
            return ",".join([str(interval) for interval in self.sub_intervals])
        return "%s-%d-%d-%d-%s" % (self.chrom, self.start, self.end, self.length, ",".join(list(self.sources)))

    def __repr__(self):
        return "<" + self.__class__.__name__ + " " + str(self.__dict__) + ">"

    def overlaps(self, other, min_fraction_self=1e-9, min_fraction_other=1e-9, min_overlap_length_self=1,
                 min_overlap_length_other=1):
        if self.chrom != other.chrom: return False
        if max(self.start - self.wiggle, other.start - other.wiggle) >= min(self.end + self.wiggle,
                                                                            other.end + other.wiggle): return False

        self_length = float(self.end - self.start + 2 * self.wiggle)
        other_length = float(other.end - other.start + 2 * other.wiggle)
        overlap_length = min(self.end + self.wiggle, other.end + other.wiggle) - max(self.start - self.wiggle,
                                                                                other.start - other.wiggle)
        return float(overlap_length) >= max(min_fraction_self * self_length,
                                            min_fraction_other * other_length) and overlap_length >= max(
            min_fraction_self, min_overlap_length_other)

    def is_adjacent(self, other, gap=0):
        if self.chrom != other.chrom: return False
        return (self.end + gap >= other.start and self.end + gap < other.end) or (
        other.end + gap >= self.start and other.end + gap < self.end)

    def get_start(self):
        if not self.sub_intervals:
            return self.start

        return min([interval.get_start() for interval in self.sub_intervals])

    def get_end(self):
        if not self.sub_intervals:
            return self.end

        return max([interval.get_end() for interval in self.sub_intervals])

    def do_validation(self, overlap_ratio=0.5):
        self.start = self.get_start()
        self.end = self.get_end()

        if self.sv_type not in svs_of_interest:
            return

        if not self.sub_intervals:
            # This interval did not overlap anything
            self.is_precise = list(self.sources)[0] in precise_sv_sources
            # if self.sv_type == "INV" and self.length >= 1000: self.is_validated = True
            return

        lists = {source: [] for source in sv_sources}

        precise_merged = merge_intervals(
            [interval for interval in self.sub_intervals if list(interval.sources)[0] in precise_sv_sources])

        for interval in self.sub_intervals:
            # logger.debug("Examining subinterval " + str(interval))
            lists[list(interval.sources)[0]].append(interval)

        for source in sv_sources:
            # If the SV is called multiple times by the same tool, don't validate
            if len(lists[source]) == 1:
                if self.overlaps(lists[source][0], overlap_ratio, overlap_ratio) \
                        or interval_overlaps_interval_list(lists[source][0], precise_merged, overlap_ratio,
                                                           overlap_ratio):
                    self.is_validated = len(self.sources) > 1
                    self.validating_interval = lists[source][0]
                    self.is_precise = source in precise_sv_sources
                    break # This break makes the order of sv_sources important

        if not self.is_validated:
            return

        if self.is_precise and self.validating_interval is not None:
            self.start = self.validating_interval.start
            self.end = self.validating_interval.end
            self.length = self.validating_interval.length
            self.gt = self.validating_interval.gt
            self.info = self.validating_interval.info

            if self.length == 0:
                for source in sv_sources:
                    if source in self.sources:
                        if lists[source] and lists[source][0].length > 0:
                            self.length = lists[source][0].length
                            break

    def fix_pos(self):
        if self.sv_type == "INS":
            mid = (self.start + self.end) / 2
            if not self.is_precise:
                self.end = self.start
                self.cipos = ["0", "%d" % (self.end - self.start)]
                # self.cipos = ["%d" % (mid - self.start), "%d" % (mid - self.end)]
            else:
                self.start = mid
                self.end = mid

    def to_vcf_record(self, fasta_handle, sample):
        if self.start <= 0: return None
        if self.sv_type not in svs_of_interest: return None

        # ignore private haplotype caller calls
        if ((not self.sub_intervals) or len(self.sources) == 1) and list(self.sources)[0] == "HaplotypeCaller":
            return None

        # formulate the INFO field
        info = {}
        if not self.sub_intervals:
            info.update(self.info)
        else:
            info.update(self.info)
            for interval in self.sub_intervals:
                #TODO: this will just overwrite the other dict entries... this should be ok for pass variants
                info.update(interval.info)
        svmethods = [sv_sources_to_type[tool] for tool in self.sources]
        svmethods.sort()
        sv_len = -self.length if self.sv_type == "DEL" else self.length
        info = {"SVLEN": sv_len,
                "SVTYPE": self.sv_type,
                "SVMETHOD": ",".join(svmethods)}
        if self.sv_type == "DEL" or self.sv_type == "DUP":
            info["END"] = self.end

        if not self.is_precise:
            info.update({"IMPRECISE": None})

        info.update({"VT":"SV"})
        info.update({"SVTOOL":"MetaSVMerge"})
        info.update({"NUM_SVMETHODS":len(self.sources)})
        info.update({"SOURCES":str(self)})
        if self.cipos:
            info.update({"CIPOS": (",".join([str(a) for a in self.cipos]))})

        vcf_record = vcf.model._Record(self.chrom,
                                       self.start,
                                       ".",
                                       fasta_handle.fetch(self.chrom, self.start - 1, self.start),
                                       ["<%s>" % (self.sv_type)],
                                       ".",
                                       "PASS" if self.is_validated else "LowQual",
                                       info,
                                       "GT",
                                       [0],
                                       [vcf.model._Call(None, sample, vcf.model.make_calldata_tuple("GT")(GT="1/1"))])
        return vcf_record

    def to_bed_interval(self, sample_name):
        if self.start <= 0: return None
        if self.sv_type not in ["DEL", "INS", "INV"]: return None
        # if not self.sub_intervals and list(self.sources)[0] == "HaplotypeCaller": return None
        #if len(self.sources) == 1 and list(self.sources)[0] == "HaplotypeCaller": return None
        end = self.end if self.sv_type != "INS" else (self.end + 1)

        return pybedtools.Interval(self.chrom, self.start, end, name="%s,%d,%s" % (
        self.sv_type, self.length, ";".join(sorted([sv_sources_to_type[tool] for tool in self.sources]))),
                                   score=str(len(self.sources)))

    def to_svp_record(self, sample_name, id_num):
        if self.start <= 0: return None
        if self.sv_type not in ["DEL", "INS", "INV"]: return None
        if not self.sub_intervals and list(self.sources)[0] == "HaplotypeCaller": return None
        if len(self.sources) == 1 and list(self.sources)[0] == "HaplotypeCaller": return None

        start_outer = self.start
        start_inner = self.start
        end_inner = self.end
        end_outer = self.end
        type_of_computational_approach = ",".join(sorted([sv_sources_to_type[tool] for tool in self.sources]))

        return "%s\t%d\t%d\t%d\t%d\t%s\t%d\t%s\t%s\t%s\t%s\tMetaSV\t%d" % (
        self.chrom, start_outer, start_inner, end_inner, end_outer, self.sv_type, self.length, "BWA", "Illumina",
        sample_name, type_of_computational_approach, id_num)

def interval_overlaps_interval_list(interval, interval_list, min_fraction_self=1e-9, min_fraction_other=1e-9):
    index = bisect.bisect_left(interval_list, interval)
    if index > 0 and interval.overlaps(interval_list[index - 1], min_fraction_self, min_fraction_other,
                                       min_overlap_length_self=1, min_overlap_length_other=1):
        return True
    if index < len(interval_list) and interval.overlaps(interval_list[index], min_fraction_self, min_fraction_other,
                                                        min_overlap_length_self=1, min_overlap_length_other=1):
        return True
    return False

def merge_intervals(interval_list):
    if not interval_list: return []

    interval_list.sort()
    merged_intervals = []

    current_merged_interval = copy.deepcopy(interval_list[0])
    for i in xrange(len(interval_list) - 1):
        next_interval = interval_list[i + 1]
        if current_merged_interval.overlaps(next_interval) or current_merged_interval.is_adjacent(next_interval):
            if current_merged_interval.sub_intervals:
                current_merged_interval.merge(next_interval)
            else:
                new_merged_interval = SVInterval()
                # logger.debug("Merging %s with %s" % (repr(current_merged_interval), repr(next_interval)))
                new_merged_interval.set_merged(current_merged_interval, next_interval)
                current_merged_interval = new_merged_interval
        else:
            merged_intervals.append(current_merged_interval)
            current_merged_interval = copy.deepcopy(next_interval)
    merged_intervals.append(current_merged_interval)
    merged_intervals.sort()
    return merged_intervals
<|endoftext|>