<commit_before>#!/usr/bin/env python

import jsk_arc2017_common
from jsk_arc2017_common.msg import WeightStamped
from jsk_recognition_msgs.msg import Label
from jsk_recognition_msgs.msg import LabelArray
from jsk_topic_tools import ConnectionBasedTransport
import message_filters
import rospy
from std_srvs.srv import Trigger
from std_srvs.srv import TriggerResponse

class WeightCanditatesRefiner(ConnectionBasedTransport):

    def __init__(self):
        super(WeightCanditatesRefiner, self).__init__()
        # {object_name: object_id}
        self.candidates = {}
        self.input_topics = rospy.get_param('~input_topics')
        # {object_name: weight}
        self.object_weights = jsk_arc2017_common.get_object_weights()
        self.error = rospy.get_param('~error', 1.0)

        self.weight_sum_at_reset = 0.0
        self.prev_weight_values = [0] * len(self.input_topics)

        self.weight_sum_pub = self.advertise(
            '~debug/weight_sum', WeightStamped, queue_size=1)
        self.weight_sum_at_reset_pub = self.advertise(
            '~debug/weight_sum_at_reset', WeightStamped, queue_size=1)
        self.picked_pub = self.advertise(
            '~output/candidates/picked', LabelArray, queue_size=1)
        self.placed_pub = self.advertise(
            '~output/candidates/placed', LabelArray, queue_size=1)
        self.init_srv = rospy.Service('~reset', Trigger, self._reset)

        # XXX: this node must always subscribe weight scale values
        rospy.set_param('~always_subscribe', True)
        self.subscribe()

    def subscribe(self):
        use_async = rospy.get_param('~approximate_sync', False)
        queue_size = rospy.get_param('~queue_size', 10)
        # add candidates subscriber
        self.sub_candidates = rospy.Subscriber(
            '~input/candidates', LabelArray, self._candidates_cb)
        # add scale subscriber
        self.subs = []
        for input_topic in self.input_topics:
            sub = message_filters.Subscriber(input_topic, WeightStamped)
            self.subs.append(sub)
        if use_async:
            slop = rospy.get_param('~slop', 0.1)
            sync = message_filters.ApproximateTimeSynchronizer(
                self.subs, queue_size=queue_size, slop=slop)
        else:
            sync = message_filters.TimeSynchronizer(
                self.subs, queue_size=queue_size)
        sync.registerCallback(self._scale_cb)

    def unsubscribe(self):
        self.sub_candidates.unregister()
        for sub in self.subs:
            sub.unregister()

    def _candidates_cb(self, labels_msg):
        self.candidates = {}
        for label_msg in labels_msg.labels:
            self.candidates[label_msg.name] = label_msg.id

    def _scale_cb(self, *weight_msgs):
        if not self.candidates:
            rospy.logwarn_throttle(10, 'No candidates, so skipping')
            return
        candidates = self.candidates

        assert len(weight_msgs) == len(self.prev_weight_values)
        weight_values = [w.weight.value for w in weight_msgs]
        if any(w < 0 for w in weight_values):
            return  # unstable, scale over, or something
        self.prev_weight_values = weight_values

        weight_sum = sum(weight_values)
        weight_diff = weight_sum - self.weight_sum_at_reset
        sum_msg = WeightStamped()
        sum_msg.header = weight_msgs[0].header
        sum_msg.weight.value = weight_sum
        sum_at_reset_msg = WeightStamped()
        sum_at_reset_msg.header = weight_msgs[0].header
        sum_at_reset_msg.weight.value = self.weight_sum_at_reset

        pick_msg = LabelArray()
        place_msg = LabelArray()
        pick_msg.header = weight_msgs[0].header
        place_msg.header = weight_msgs[0].header
        diff_lower = weight_diff - self.error
        diff_upper = weight_diff + self.error
        for obj_name, w in self.object_weights.items():
            if obj_name not in candidates:
                continue
            obj_id = candidates[obj_name]
            if diff_upper > w and w > diff_lower:
                label = Label()
                label.id = obj_id
                label.name = obj_name
                place_msg.labels.append(label)
            elif diff_upper > -w and -w > diff_lower:
                label = Label()
                label.id = obj_id
                label.name = obj_name
                pick_msg.labels.append(label)

        self.weight_sum_at_reset_pub.publish(sum_at_reset_msg)
        self.weight_sum_pub.publish(sum_msg)
        self.picked_pub.publish(pick_msg)
        self.placed_pub.publish(place_msg)

    def _reset(self, req):
        is_success = True
        try:
            self.weight_sum_at_reset = sum(self.prev_weight_values)
        except Exception:
            is_success = False
        return TriggerResponse(success=is_success)

if __name__ == '__main__':
    rospy.init_node('weight_candidates_refiner')
    app = WeightCanditatesRefiner()
    rospy.spin()
<commit_msg>No use of ConnectionBasedTransport in WeightCanditatesRefiner<commit_after>#!/usr/bin/env python

import jsk_arc2017_common
from jsk_arc2017_common.msg import WeightStamped
from jsk_recognition_msgs.msg import Label
from jsk_recognition_msgs.msg import LabelArray
from jsk_topic_tools import ConnectionBasedTransport
import message_filters
import rospy
from std_srvs.srv import Trigger
from std_srvs.srv import TriggerResponse

class WeightCanditatesRefiner(object):

    def __init__(self):
        # {object_name: object_id}
        self.candidates = {}
        self.input_topics = rospy.get_param('~input_topics')
        # {object_name: weight}
        self.object_weights = jsk_arc2017_common.get_object_weights()
        self.error = rospy.get_param('~error', 1.0)

        self.weight_sum_at_reset = 0.0
        self.prev_weight_values = [0] * len(self.input_topics)

        self.weight_sum_pub = rospy.Publisher(
            '~debug/weight_sum', WeightStamped, queue_size=1)
        self.weight_sum_at_reset_pub = rospy.Publisher(
            '~debug/weight_sum_at_reset', WeightStamped, queue_size=1)
        self.picked_pub = rospy.Publisher(
            '~output/candidates/picked', LabelArray, queue_size=1)
        self.placed_pub = rospy.Publisher(
            '~output/candidates/placed', LabelArray, queue_size=1)
        self.init_srv = rospy.Service('~reset', Trigger, self._reset)
        self.subscribe()

    def subscribe(self):
        use_async = rospy.get_param('~approximate_sync', False)
        queue_size = rospy.get_param('~queue_size', 10)
        # add candidates subscriber
        self.sub_candidates = rospy.Subscriber(
            '~input/candidates', LabelArray, self._candidates_cb)
        # add scale subscriber
        self.subs = []
        for input_topic in self.input_topics:
            sub = message_filters.Subscriber(input_topic, WeightStamped, queue_size=1)
            self.subs.append(sub)
        if use_async:
            slop = rospy.get_param('~slop', 0.1)
            sync = message_filters.ApproximateTimeSynchronizer(
                self.subs, queue_size=queue_size, slop=slop)
        else:
            sync = message_filters.TimeSynchronizer(
                self.subs, queue_size=queue_size)
        sync.registerCallback(self._scale_cb)

    # def unsubscribe(self):
    #     self.sub_candidates.unregister()
    #     for sub in self.subs:
    #         sub.unregister()

    def _candidates_cb(self, labels_msg):
        self.candidates = {}
        for label_msg in labels_msg.labels:
            self.candidates[label_msg.name] = label_msg.id

    def _scale_cb(self, *weight_msgs):
        if not self.candidates:
            rospy.logwarn_throttle(10, 'No candidates, so skipping')
            return
        candidates = self.candidates

        assert len(weight_msgs) == len(self.prev_weight_values)
        weight_values = [w.weight.value for w in weight_msgs]
        if any(w < 0 for w in weight_values):
            return  # unstable, scale over, or something
        self.prev_weight_values = weight_values

        weight_sum = sum(weight_values)
        weight_diff = weight_sum - self.weight_sum_at_reset
        sum_msg = WeightStamped()
        sum_msg.header = weight_msgs[0].header
        sum_msg.weight.value = weight_sum
        sum_at_reset_msg = WeightStamped()
        sum_at_reset_msg.header = weight_msgs[0].header
        sum_at_reset_msg.weight.value = self.weight_sum_at_reset

        pick_msg = LabelArray()
        place_msg = LabelArray()
        pick_msg.header = weight_msgs[0].header
        place_msg.header = weight_msgs[0].header
        diff_lower = weight_diff - self.error
        diff_upper = weight_diff + self.error
        for obj_name, w in self.object_weights.items():
            if obj_name not in candidates:
                continue
            obj_id = candidates[obj_name]
            if diff_upper > w and w > diff_lower:
                label = Label()
                label.id = obj_id
                label.name = obj_name
                place_msg.labels.append(label)
            elif diff_upper > -w and -w > diff_lower:
                label = Label()
                label.id = obj_id
                label.name = obj_name
                pick_msg.labels.append(label)

        self.weight_sum_at_reset_pub.publish(sum_at_reset_msg)
        self.weight_sum_pub.publish(sum_msg)
        self.picked_pub.publish(pick_msg)
        self.placed_pub.publish(place_msg)

    def _reset(self, req):
        is_success = True
        try:
            self.weight_sum_at_reset = sum(self.prev_weight_values)
        except Exception:
            is_success = False
        return TriggerResponse(success=is_success)

if __name__ == '__main__':
    rospy.init_node('weight_candidates_refiner')
    app = WeightCanditatesRefiner()
    rospy.spin()
<|endoftext|>


<commit_before>from orator.migrations import Migration

class CreateUserPermissionTable(Migration):

    def up(self):
        """
        Run the migrations.
        """
        with self.schema.create('user_groups') as table:
            table.integer('user_id').unsigned()
            table.foreign('user_id').references('id').on('users')
            table.integer('group_id').unsigned()
            table.foreign('group_id').references('id').on('groups').on_delete('cascade')
            table.primary(['user_id', 'group_id'])

    def down(self):
        """
        Revert the migrations.
        """
        self.schema.drop('user_groups')
<commit_msg>* auth: fix migration<commit_after>from orator.migrations import Migration

class CreateUserGroupsTable(Migration):

    def up(self):
        """
        Run the migrations.
        """
        with self.schema.create('user_groups') as table:
            table.integer('user_id').unsigned()
            table.foreign('user_id').references('id').on('users')
            table.integer('group_id').unsigned()
            table.foreign('group_id').references('id').on('groups').on_delete('cascade')
            table.primary(['user_id', 'group_id'])

    def down(self):
        """
        Revert the migrations.
        """
        self.schema.drop('user_groups')
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
# encoding: utf-8
"""
survey/core.py

Created by Karl Dubost on 2012-07-03.
Copyright (c) 2011 Grange. All rights reserved.
see LICENSE
"""

import urllib2
import urlparse
import requests
from lxml import etree
import cssutils
import os.path
import logging
from datetime import datetime

# CONSTANT - ALL OF THESE will have to be passed as arguements.
# List of URIs, 1 URI by line
#SITELIST = os.path.dirname(__file__) + "/../tests/urlist-top100.data"
SITELIST = os.path.dirname(__file__) + "/../tests/urlist.data"
# UAREF = "Opera/9.80 (Macintosh; Intel Mac OS X 10.8.0) Presto/2.12.363 Version/12.50"
UAREF = "Opera/9.80 (Android 2.3.5; Linux; Opera Mobi/ADR-1202082305; U; en) Presto/2.10.254 Version/12.00"
UATEST = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6"
logging.basicConfig(filename='log_filename.txt', level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')

class UriCheck:
    """Some control about URIs from the file"""

    def __init__(self):
        self.uri = ""

    def ishttpURI(self, uri):
        """Given a URI,
        returns True if http or https
        returns False if not http or https"""
        req = urllib2.Request(uri)
        if req.get_type() in ['http', 'https']:
            return True
        else:
            return False

class HttpRequests:
    """Servers behave depending on what is asking which URI"""

    def __init__(self):
        self.content = ""
        self.statuscode = ""

    def getRequest(self, uri, uastring):
        """Given a URI and a UA String,
        returns a list with uri, newlocation, statuscode, content"""
        headers = {'User-Agent': uastring}
        r = requests.get(uri, headers=headers)
        statuscode = r.status_code
        responseheaders = r.headers
        history = r.history
        return statuscode, responseheaders, history

    def getContent(self, uri, useragentstring):
        """Return the content associated with an uri"""
        headers = {'User-Agent': useragentstring}
        r = requests.get(uri, headers=headers)
        responsetext = r.text
        # first test if it's a string
        try:
            isinstance(responsetext, basestring)
        # Log type errors
        except TypeError as e:
            logging.debug("HTMLTEXT - %s" % (e.message))
        else:
            # then test if it's unicode and convert it
            if isinstance(responsetext, unicode):
                htmltext = responsetext.encode('utf-8')
            # or if not it is just the common text
            elif isinstance(responsetext, str):
                htmltext = responsetext
        return htmltext

class Css:
    """Grabing All CSS for one given URI"""

    def __init__(self):
        self.htmltext = ""

    def getCssUriList(self, htmltext, uri):
        """Given an htmltext, get the list of linked CSS"""
        cssurilist = []
        tree = etree.HTML(htmltext)
        csspathlist = tree.xpath('//link[@rel="stylesheet"]/@href')
        for i, csspath in enumerate(csspathlist):
            cssurl = urlparse.urljoin(uri, csspath)
            cssurilist.append(cssurl)
        return cssurilist

    def getCssHttpUriList(self, httpheadersdict, uri):
        """Given HTTP headers dictionary for a URI, extract the list of linked CSS"""
        # parse HTTP header for Link: <uri>;rel=stylesheet
        # requests returns something such as:
        # 'link': '</foo.css>;rel=stylesheet, </>;rel=next'
        # httplib might return a list of csv for a header.
        cssurilist = []
        req = HttpRequests()
        linkhttp = httpheadersdict['link']
        csspathlist = req.parseHttpLink(linkhttp)
        for i, csspath in enumerate(csspathlist):
            cssurl = urlparse.urljoin(uri, csspath)
            cssurilist.append(cssurl)
        return cssurilist

    def getCssRules(self, uri):
        """Given the URI of a CSS file,
        return the list of all CSS rules,
        including all the imports."""
        IMPORT_FLAG = False
        stylesheet = cssutils.parseUrl(uri)
        # Check if there is an import rule in the CSS
        for cssrule in stylesheet:
            if cssrule.type == 3:
                IMPORT_FLAG = True
        # ResolveImports only if there is a rule
        if IMPORT_FLAG:
            stylesheet2 = cssutils.resolveImports(stylesheet)
        else:
            stylesheet2 = stylesheet
        return stylesheet2

    def hasStyleElement(self, htmltext):
        """Given an htmltext,
        check if the style element is available
        return True if yes"""
        tree = etree.HTML(htmltext)
        styleelements = tree.xpath('//style')
        if not styleelements:
            return False
        else:
            return True

    def getStyleElementRules(self, htmltext):
        """Given an htmltext,
        return the CSS rules contained in the content"""
        compiledstyle = ""
        stylesheet = cssutils.css.CSSStyleSheet()
        tree = etree.HTML(htmltext)
        styleelements = tree.xpath('//style')
        for styleelt in styleelements:
            if styleelt.text != None:
                compiledstyle = compiledstyle + styleelt.text
            else:
                logging.debug("STYLE ELEMENT %s on %s" % (styleelements.index(styleelt) + 1, len(styleelements)))
        if compiledstyle != None:
            cssutils.ser.prefs.indentClosingBrace = False
            cssutils.ser.prefs.keepComments = False
            cssutils.ser.prefs.lineSeparator = u''
            cssutils.ser.prefs.omitLastSemicolon = False
            try:
                stylesheet = cssutils.parseString(compiledstyle)
            except ValueError as e:
                logging.info("BOGUS STYLE RULE: %s" % (e.message))
        else:
            raise ValueError("STYLE ELEMENT: no CSS Rules")
        return stylesheet

    def hasVendorProperty(self, vendorname, propertyname, declarationslist):
        """Given a declarationslist, a vendor name, and propertyname,
        verify that the the property name for this vendor name exists
        return True"""
        expectedproperty = "-%s-%s" % (vendorname, propertyname)
        propertynamelist = declarationslist.keys()
        try:
            propertynamelist.index(expectedproperty)
            return True
        except ValueError:
            return False

    def hasProperty(self, propertyname, declarationslist):
        """Given a declarationslist, a propertyname,
        verify that the the property name exists
        return True"""
        propertynamelist = declarationslist.keys()
        try:
            propertynamelist.index(propertyname)
            return True
        except ValueError:
            return False

class Survey:
    """Mother of survey modules"""

    def __init__(self):
        # TODO: Initializing the reporting output mode
        pass

    def compareCssProperties(self, vendorref, vendorcomp, propertyname, cssrule):
        """compares how CSS properties are used and computes a score
        vendorref  = +1
        vendorcomp = +2
        prefixless = +4

        Example:
        .sel {  -o-color: #fff;
              -moz-color: #fff;
                   color: #fff;}

        will return a score of 5
        with property = 'color', vendorref = 'o', vendorcomp = 'moz'
        """
        score = 0
        propertyvendorlist = []
        propertyvendorlist.append(propertyname)
        css = Css()
        if css.hasVendorProperty(vendorref, propertyname, cssrule):
            score += 1
            propertyvendorlist.append(vendorref)
        if css.hasVendorProperty(vendorcomp, propertyname, cssrule):
            score += 2
            propertyvendorlist.append(vendorcomp)
        if css.hasProperty(propertyname, cssrule):
            score += 4
            propertyvendorlist.append('prefixless')
        return score, propertyvendorlist

def main():
    cssutils.log.setLevel(logging.FATAL)
    uc = UriCheck()
    req = HttpRequests()
    css = Css()
    survey = Survey()
    propertytocheck = ['transition', 'transform', 'animation']
    with open(SITELIST) as f:
        logging.info("START - %s with %s" % (datetime.today().isoformat(), UAREF))
        for uri in f:
            # remove leading, trailing spaces
            if uri.startswith("#"):
                continue
            uri = uri.strip()
            logging.info("SITE: %s" % (uri))
            if uc.ishttpURI(uri):
                try:
                    htmltext = req.getContent(uri, UAREF)
                    if css.hasStyleElement(htmltext):
                        styleeltrule = css.getStyleElementRules(htmltext)
                        if styleeltrule != "":
                            logging.info("Style Element at %s" % (uri))
                            for cssrule in styleeltrule:
                                if cssrule.type == 1:
                                    for i, propertyname in enumerate(propertytocheck):
                                        score, propertyresultlist = survey.compareCssProperties('o', 'webkit', propertyname, cssrule.style)
                                        # printing only if the propertyname is found and if there is more than prefixless
                                        if score > 0 and score != 4:
                                            print propertyresultlist, uri

                    cssurislist = css.getCssUriList(htmltext, uri)
                    if len(cssurislist) == 0:
                        logging.info("NO CSS LINK REL at %s" % (uri))
                    else:
                        logging.info("CSS LINK REL: %s at %s" % (len(cssurislist), uri))

                    for cssuri in cssurislist:
                        cssruleslist = css.getCssRules(cssuri)
                        try:
                            cssruleslist = css.getCssRules(cssuri)
                        except ValueError as e:
                            logging.info("CSS Mime Type error: %s" % (e.message))
                        for cssrule in cssruleslist:
                            if cssrule.type == 1:
                                # This is a rule, we process
                                # Reminder cssrule.type == 0 -> comment
                                for i, propertyname in enumerate(propertytocheck):
                                    score, propertyresultlist = survey.compareCssProperties('o', 'webkit', propertyname, cssrule.style)
                                    # printing only if the propertyname is found and if there is more than prefixless
                                    # if score > 0 and score != 4:
                                    if score > 0:
                                        print uri, propertyresultlist
                                        logging.info("SURVEY: %s %s" % (uri, propertyresultlist))
                        # logging.info(responseheaders)

                except requests.exceptions.ConnectionError as e:
                    # we are catching network connection error and record them.
                    logging.info("Connection error: %s" % (e.message))

if __name__ == '__main__':
    main()
<commit_msg>after redirections we need to return the last url of the content for other resources<commit_after>#!/usr/bin/env python2.7
# encoding: utf-8
"""
survey/core.py

Created by Karl Dubost on 2012-07-03.
Copyright (c) 2011 Grange. All rights reserved.
see LICENSE
"""

import urllib2
import urlparse
import requests
from lxml import etree
import cssutils
import os.path
import logging
from datetime import datetime

# CONSTANT - ALL OF THESE will have to be passed as arguements.
# List of URIs, 1 URI by line
#SITELIST = os.path.dirname(__file__) + "/../tests/urlist-top100.data"
SITELIST = os.path.dirname(__file__) + "/../tests/urlist.data"
# UAREF = "Opera/9.80 (Macintosh; Intel Mac OS X 10.8.0) Presto/2.12.363 Version/12.50"
UAREF = "Opera/9.80 (Android 2.3.5; Linux; Opera Mobi/ADR-1202082305; U; en) Presto/2.10.254 Version/12.00"
UATEST = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6"
logging.basicConfig(filename='log_filename.txt', level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')

class UriCheck:
    """Some control about URIs from the file"""

    def __init__(self):
        self.uri = ""

    def ishttpURI(self, uri):
        """Given a URI,
        returns True if http or https
        returns False if not http or https"""
        req = urllib2.Request(uri)
        if req.get_type() in ['http', 'https']:
            return True
        else:
            return False

class HttpRequests:
    """Servers behave depending on what is asking which URI"""

    def __init__(self):
        self.content = ""
        self.statuscode = ""

    def getRequest(self, uri, uastring):
        """Given a URI and a UA String,
        returns a list with uri, newlocation, statuscode, content"""
        headers = {'User-Agent': uastring}
        r = requests.get(uri, headers=headers)
        statuscode = r.status_code
        responseheaders = r.headers
        history = r.history
        return statuscode, responseheaders, history

    def getContent(self, uri, useragentstring):
        """Return the content associated with an uri"""
        headers = {'User-Agent': useragentstring}
        r = requests.get(uri, headers=headers)
        # r.history contains a list of the different redirections
        # if needed later.
        finaluri = r.url
        responsetext = r.text
        # first test if it's a string
        try:
            isinstance(responsetext, basestring)
        # Log type errors
        except TypeError as e:
            logging.debug("HTMLTEXT - %s" % (e.message))
        else:
            # then test if it's unicode and convert it
            if isinstance(responsetext, unicode):
                htmltext = responsetext.encode('utf-8')
            # or if not it is just the common text
            elif isinstance(responsetext, str):
                htmltext = responsetext
        return htmltext, finaluri

class Css:
    """Grabing All CSS for one given URI"""

    def __init__(self):
        self.htmltext = ""

    def getCssUriList(self, htmltext, uri):
        """Given an htmltext, get the list of linked CSS"""
        cssurilist = []
        tree = etree.HTML(htmltext)
        csspathlist = tree.xpath('//link[@rel="stylesheet"]/@href')
        for i, csspath in enumerate(csspathlist):
            cssurl = urlparse.urljoin(uri, csspath)
            cssurilist.append(cssurl)
        return cssurilist

    def getCssHttpUriList(self, httpheadersdict, uri):
        """Given HTTP headers dictionary for a URI, extract the list of linked CSS"""
        # parse HTTP header for Link: <uri>;rel=stylesheet
        # requests returns something such as:
        # 'link': '</foo.css>;rel=stylesheet, </>;rel=next'
        # httplib might return a list of csv for a header.
        cssurilist = []
        req = HttpRequests()
        linkhttp = httpheadersdict['link']
        csspathlist = req.parseHttpLink(linkhttp)
        for i, csspath in enumerate(csspathlist):
            cssurl = urlparse.urljoin(uri, csspath)
            cssurilist.append(cssurl)
        return cssurilist

    def getCssRules(self, uri):
        """Given the URI of a CSS file,
        return the list of all CSS rules,
        including all the imports."""
        IMPORT_FLAG = False
        stylesheet = cssutils.parseUrl(uri)
        # Check if there is an import rule in the CSS
        for cssrule in stylesheet:
            if cssrule.type == 3:
                IMPORT_FLAG = True
        # ResolveImports only if there is a rule
        if IMPORT_FLAG:
            stylesheet2 = cssutils.resolveImports(stylesheet)
        else:
            stylesheet2 = stylesheet
        return stylesheet2

    def hasStyleElement(self, htmltext):
        """Given an htmltext,
        check if the style element is available
        return True if yes"""
        tree = etree.HTML(htmltext)
        styleelements = tree.xpath('//style')
        if not styleelements:
            return False
        else:
            return True

    def getStyleElementRules(self, htmltext):
        """Given an htmltext,
        return the CSS rules contained in the content"""
        compiledstyle = ""
        stylesheet = cssutils.css.CSSStyleSheet()
        tree = etree.HTML(htmltext)
        styleelements = tree.xpath('//style')
        for styleelt in styleelements:
            if styleelt.text != None:
                compiledstyle = compiledstyle + styleelt.text
            else:
                logging.debug("STYLE ELEMENT %s on %s" % (styleelements.index(styleelt) + 1, len(styleelements)))
        if compiledstyle != None:
            cssutils.ser.prefs.indentClosingBrace = False
            cssutils.ser.prefs.keepComments = False
            cssutils.ser.prefs.lineSeparator = u''
            cssutils.ser.prefs.omitLastSemicolon = False
            try:
                stylesheet = cssutils.parseString(compiledstyle)
            except ValueError as e:
                logging.info("BOGUS STYLE RULE: %s" % (e.message))
        else:
            raise ValueError("STYLE ELEMENT: no CSS Rules")
        return stylesheet

    def hasVendorProperty(self, vendorname, propertyname, declarationslist):
        """Given a declarationslist, a vendor name, and propertyname,
        verify that the the property name for this vendor name exists
        return True"""
        expectedproperty = "-%s-%s" % (vendorname, propertyname)
        propertynamelist = declarationslist.keys()
        try:
            propertynamelist.index(expectedproperty)
            return True
        except ValueError:
            return False

    def hasProperty(self, propertyname, declarationslist):
        """Given a declarationslist, a propertyname,
        verify that the the property name exists
        return True"""
        propertynamelist = declarationslist.keys()
        try:
            propertynamelist.index(propertyname)
            return True
        except ValueError:
            return False

class Survey:
    """Mother of survey modules"""

    def __init__(self):
        # TODO: Initializing the reporting output mode
        pass

    def compareCssProperties(self, vendorref, vendorcomp, propertyname, cssrule):
        """compares how CSS properties are used and computes a score
        vendorref  = +1
        vendorcomp = +2
        prefixless = +4

        Example:
        .sel {  -o-color: #fff;
              -moz-color: #fff;
                   color: #fff;}

        will return a score of 5
        with property = 'color', vendorref = 'o', vendorcomp = 'moz'
        """
        score = 0
        propertyvendorlist = []
        propertyvendorlist.append(propertyname)
        css = Css()
        if css.hasVendorProperty(vendorref, propertyname, cssrule):
            score += 1
            propertyvendorlist.append(vendorref)
        if css.hasVendorProperty(vendorcomp, propertyname, cssrule):
            score += 2
            propertyvendorlist.append(vendorcomp)
        if css.hasProperty(propertyname, cssrule):
            score += 4
            propertyvendorlist.append('prefixless')
        return score, propertyvendorlist

def main():
    cssutils.log.setLevel(logging.FATAL)
    uc = UriCheck()
    req = HttpRequests()
    css = Css()
    survey = Survey()
    propertytocheck = ['transition', 'transform', 'animation']
    with open(SITELIST) as f:
        logging.info("START - %s with %s" % (datetime.today().isoformat(), UAREF))
        for uri in f:
            # remove leading, trailing spaces
            if uri.startswith("#"):
                continue
            uri = uri.strip()
            logging.info("SITE: %s" % (uri))
            if uc.ishttpURI(uri):
                try:
                    htmltext = req.getContent(uri, UAREF)
                    if css.hasStyleElement(htmltext):
                        styleeltrule = css.getStyleElementRules(htmltext)
                        if styleeltrule != "":
                            logging.info("Style Element at %s" % (uri))
                            for cssrule in styleeltrule:
                                if cssrule.type == 1:
                                    for i, propertyname in enumerate(propertytocheck):
                                        score, propertyresultlist = survey.compareCssProperties('o', 'webkit', propertyname, cssrule.style)
                                        # printing only if the propertyname is found and if there is more than prefixless
                                        if score > 0 and score != 4:
                                            print propertyresultlist, uri

                    cssurislist = css.getCssUriList(htmltext, uri)
                    if len(cssurislist) == 0:
                        logging.info("NO CSS LINK REL at %s" % (uri))
                    else:
                        logging.info("CSS LINK REL: %s at %s" % (len(cssurislist), uri))

                    for cssuri in cssurislist:
                        cssruleslist = css.getCssRules(cssuri)
                        try:
                            cssruleslist = css.getCssRules(cssuri)
                        except ValueError as e:
                            logging.info("CSS Mime Type error: %s" % (e.message))
                        for cssrule in cssruleslist:
                            if cssrule.type == 1:
                                # This is a rule, we process
                                # Reminder cssrule.type == 0 -> comment
                                for i, propertyname in enumerate(propertytocheck):
                                    score, propertyresultlist = survey.compareCssProperties('o', 'webkit', propertyname, cssrule.style)
                                    # printing only if the propertyname is found and if there is more than prefixless
                                    # if score > 0 and score != 4:
                                    if score > 0:
                                        print uri, propertyresultlist
                                        logging.info("SURVEY: %s %s" % (uri, propertyresultlist))
                        # logging.info(responseheaders)

                except requests.exceptions.ConnectionError as e:
                    # we are catching network connection error and record them.
                    logging.info("Connection error: %s" % (e.message))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import subprocess
from hierarchy import *
from tikzify import *

formulae = 'lem wlem dgp glpo glpoa gmp wgmp dp he dpn hen dnsu dnse ud ip'.split()
globals().update({f: f for f in formulae})
efq = 'efq'

globals().update({future: future for future in
 'ud udn lem wlem dp dpn he dnsu dnse glpon glpoa gmp mgmp dgp'.split()})

# These are actually equivalent. Condense them once investigations are complete.
# glpo = lem
# hen = dpn
# wgmp = dnse

formula_layout = '''\
    glpoa
                              lem       glpo
                dp                             he
                                    dpn        hen
                ud  gmp            dgp
        wgmp
            dnsu       glpon                    dnse
                              wlem
'''

formula_strs = {f: f.upper() for f in formulae}
formula_strs[dnse] = R'DNS$\exists$'
formula_strs[glpoa] = "GLPO$'$"
formula_strs[glpon] = R'GLPO$_\neg$'

formula_strs[dnsu] = R'DNS$\forall$,WGMP'
formula_strs[lem] = R'LEM,GLPO'
formula_strs[dpn] = R'DP$_\lnot$,HE$_\lnot$'

unnamed_proofs = {
    (he, ip), (ip, he),
    (lem, glpo), (glpo, lem),
    (dpn, hen), (hen, dpn),
    (dnsu, wgmp), (wgmp, dnsu),
    (lem, wlem),
    (dp, dpn),
    (he, hen),
    (gmp, wgmp),
    (dgp, wlem),
    (glpoa, lem),
    (glpoa, gmp),
    (dp, dnsu),
    (glpo, dpn),
    (he, dnse),
    (glpo, dnse),
    (gmp, dnse),
    (dpn, dnse),
    (glpoa, wgmp),
}

proofs = {p: '{}-{}'.format(','.join(p[:-1]), p[-1]) for p in unnamed_proofs}

named_models = {
    'dp-cm': (
        {efq, he, dgp, wlem, glpon, ud},
        {dp, lem, dnsu, wgmp, mgmp},
    ),
    'dp-cm-lobot': (
        {he, lem, dpn, hen, dgp, wlem, dnsu, dnse, glpo, glpoa, glpon, gmp, ud},
        {dp},
    ),
    'he-cm': (
        {efq, dp, dgp, wlem, glpon, ud},
        {he, lem},
    ),
    'he-cm-lobot': (
        {dp, lem, dpn, hen, dgp, wlem, dnsu, dnse, glpo, glpoa, glpon, gmp, ud},
        {he},
    ),
    'linear-growing-terms': (
        {efq, wlem, dgp},
        {dp, he, lem, dnse, glpoa, ud},
    ),
    'two-world-constant-terms': (
        {efq, dp, he, wlem, dgp, ud},
        {lem},
    ),
    'two-world-growing-terms': (
        {efq, wlem, dgp, wgmp},
        {glpoa, dp, he, dpn, hen, gmp, dnse, glpon, ud},
    ),
    'two-world-growing-terms-lobot': (
        {gmp, glpoa},
        {ud},
    ),
    'two-world-growing-terms-with-bot': (
        {lem, wlem, dgp},
        {glpoa, dp, he, gmp, wgmp, ud, mgmp},
    ),
    'v-const-term': (
        {efq, dnsu, ud},
        {wlem, dgp, dnse},
    ),
    'v-const-term-lobot': (
        {glpoa, lem, dpn, hen, gmp, dnse, glpon, ud},
        {dgp},
    ),
    'diamond-constant-terms': (
        {efq, wlem, gmp, ud},
        {dgp, lem},
    ),
    'beth-width-two': (
        {lem, he, dp},
        set(),
    ),
    'one-term-v': (
        {dp, he},
        {wlem, dgp},
    ),
    'trivial-lobot': (
        {f for f in formulae if f is not efq},
        {efq},
    ),
}
models = [(k, *map(frozenset, v)) for k, v in named_models.items()]

if __name__ == '__main__':
    possible_edges = find_evaluated_connections(formulae, set(proofs), list(models))
    minimal_diagram = TikzHierarchy(name_dict=formula_strs)
    minimal_diagram.add_string_node_layout(formula_layout)
    minimal_diagram.add_edges(spanning_tree(set(proofs)), color=False)
    minimal_diagram.add_edges(set(possible_edges), 'dashed')

    efq_diagram = TikzHierarchy(minimal_diagram)
    efq_proofs = {(lem, efq, f): 'classical' for f in formulae}
    efq_diagram.add_edges(spanning_tree(set(efq_proofs), set(proofs)), 'dotted')

    int_proofs = dict(proofs)
    int_models = {m for m in models if efq in m[1]}
    int_proofs.update({(lem, f): 'classical' for f in formulae})
    int_possible_edges = find_possible_connections(
                         formulae, int_proofs, int_models)
    int_diagram = TikzHierarchy(name_dict=formula_strs)
    int_diagram.add_string_node_layout(formula_layout)
    int_diagram.add_edges(spanning_tree(set(int_proofs)), color=False)
    int_diagram.add_edges(int_possible_edges, 'dashed')

    two_possible_edges = find_evaluated_connections(
                             formulae, set(proofs), list(models), free=(), order=2)
    two_diagram = TikzHierarchy(name_dict=formula_strs)
    two_diagram.add_string_node_layout(formula_layout)
    two_diagram.add_edges(spanning_tree(set(proofs)), color=False)
    two_diagram.add_edges(set(two_possible_edges), 'dashed')

    tex = make_sections(
        ('Minimal Logic', minimal_diagram),
        ('Investigations ({})'.format(len(possible_edges)),
            make_columns(make_connections_list(possible_edges)), 1),
        ('Minimal Logic with EFQ links', efq_diagram),
        ('Intuitionistic Logic', int_diagram),
        ('Minimal Logic Two-premise Possibilities', two_diagram),
        ('Investigations ({})'.format(len(two_possible_edges)),
            make_columns(make_connections_list(two_possible_edges)), 1),
    )

    document = make_latex_document(tex)

    with open('drinker.tex', 'w') as f:
        f.write(document)
    subprocess.call(['pdflatex', 'drinker.tex'], stdout=subprocess.DEVNULL)
    #with open('backdrinker.tex', 'r') as f:
    #    assert(f.read() == document)
<commit_msg>Update<commit_after>import subprocess
from hierarchy import *
from tikzify import *

formulae = 'lem wlem dgp glpo glpoa gmp wgmp dp he dpn hen dnsu dnse ud ip'.split()
globals().update({f: f for f in formulae})
efq = 'efq'

globals().update({future: future for future in
 'ud udn lem wlem dp dpn he dnsu dnse glpon glpoa gmp mgmp dgp'.split()})

# These are actually equivalent. Condense them once investigations are complete.
# glpo = lem
# hen = dpn
# wgmp = dnse

formula_layout = '''\
    glpoa
                              lem       glpo
                dp                             he
                                    dpn        hen
                ud  gmp            dgp
        wgmp
            dnsu       glpon                    dnse
                              wlem
'''

formula_strs = {f: f.upper() for f in formulae}
formula_strs[dnse] = R'DNS$\exists$'
formula_strs[glpoa] = "GLPO$'$"
formula_strs[glpon] = R'GLPO$_\neg$'

formula_strs[dnsu] = R'DNS$\forall$,WGMP'
formula_strs[lem] = R'LEM,GLPO'
formula_strs[dpn] = R'DP$_\lnot$,HE$_\lnot$'

unnamed_proofs = {
    (he, ip), (ip, he),
    (lem, glpo), (glpo, lem),
    (dpn, hen), (hen, dpn),
    (dnsu, wgmp), (wgmp, dnsu),
    (lem, wlem),
    (dp, dpn),
    (he, hen),
    (gmp, wgmp),
    (dgp, wlem),
    (glpoa, lem),
    (glpoa, gmp),
    (dp, gmp),
#    (dp, dnsu),
    (glpo, dpn),
    (he, dnse),
    (glpo, dnse),
    (gmp, dnse),
    (dpn, dnse),
    (glpoa, wgmp),
}

proofs = {p: '{}-{}'.format(','.join(p[:-1]), p[-1]) for p in unnamed_proofs}

named_models = {
    'dp-cm': (
        {efq, he, dgp, wlem, glpon, ud},
        {dp, lem, dnsu, wgmp, mgmp},
    ),
    'dp-cm-lobot': (
        {he, lem, dpn, hen, dgp, wlem, dnsu, dnse, glpo, glpoa, glpon, gmp, ud},
        {dp},
    ),
    'he-cm': (
        {efq, dp, dgp, wlem, glpon, ud},
        {he, lem},
    ),
    'he-cm-lobot': (
        {dp, lem, dpn, hen, dgp, wlem, dnsu, dnse, glpo, glpoa, glpon, gmp, ud},
        {he},
    ),
    'linear-growing-terms': (
        {efq, wlem, dgp},
        {dp, he, lem, dnse, glpoa, ud},
    ),
    'two-world-constant-terms': (
        {efq, dp, he, wlem, dgp, ud},
        {lem},
    ),
    'two-world-growing-terms': (
        {efq, wlem, dgp, wgmp},
        {glpoa, dp, he, dpn, hen, gmp, dnse, glpon, ud},
    ),
    'two-world-growing-terms-lobot': (
        {gmp, glpoa},
        {ud},
    ),
    'two-world-growing-terms-with-bot': (
        {lem, wlem, dgp},
        {glpoa, dp, he, gmp, wgmp, ud, mgmp},
    ),
    'v-const-term': (
        {efq, dnsu, ud},
        {wlem, dgp, dnse},
    ),
    'v-const-term-lobot': (
        {glpoa, lem, dpn, hen, gmp, dnse, glpon, ud},
        {dgp},
    ),
    'diamond-constant-terms': (
        {efq, wlem, gmp, ud},
        {dgp, lem},
    ),
    'beth-width-two': (
        {lem, he, dp},
        set(),
    ),
    'one-term-v': (
        {dp, he},
        {wlem, dgp},
    ),
    'trivial-lobot': (
        {f for f in formulae if f is not efq},
        {efq},
    ),
}
models = [(k, *map(frozenset, v)) for k, v in named_models.items()]

if __name__ == '__main__':
    possible_edges = find_evaluated_connections(formulae, set(proofs), list(models))
    minimal_diagram = TikzHierarchy(name_dict=formula_strs)
    minimal_diagram.add_string_node_layout(formula_layout)
    minimal_diagram.add_edges(spanning_tree(set(proofs)), color=False)
    minimal_diagram.add_edges(set(possible_edges), 'dashed')

    efq_diagram = TikzHierarchy(minimal_diagram)
    efq_proofs = {(lem, efq, f): 'classical' for f in formulae}
    efq_diagram.add_edges(spanning_tree(set(efq_proofs), set(proofs)), 'dotted')

    int_proofs = dict(proofs)
    int_models = {m for m in models if efq in m[1]}
    int_proofs.update({(lem, f): 'classical' for f in formulae})
    int_possible_edges = find_possible_connections(
                         formulae, int_proofs, int_models)
    int_diagram = TikzHierarchy(name_dict=formula_strs)
    int_diagram.add_string_node_layout(formula_layout)
    int_diagram.add_edges(spanning_tree(set(int_proofs)), color=False)
    int_diagram.add_edges(int_possible_edges, 'dashed')

    two_possible_edges = find_evaluated_connections(
                             formulae, set(proofs), list(models), free=(), order=2)
    two_diagram = TikzHierarchy(name_dict=formula_strs)
    two_diagram.add_string_node_layout(formula_layout)
    two_diagram.add_edges(spanning_tree(set(proofs)), color=False)
    two_diagram.add_edges(set(two_possible_edges), 'dashed')

    tex = make_sections(
        ('Minimal Logic', minimal_diagram),
        ('Investigations ({})'.format(len(possible_edges)),
            make_columns(make_connections_list(possible_edges)), 1),
        ('Minimal Logic with EFQ links', efq_diagram),
        ('Intuitionistic Logic', int_diagram),
        ('Minimal Logic Two-premise Possibilities', two_diagram),
        ('Investigations ({})'.format(len(two_possible_edges)),
            make_columns(make_connections_list(two_possible_edges)), 1),
    )

    document = make_latex_document(tex)

    with open('drinker.tex', 'w') as f:
        f.write(document)
    subprocess.call(['pdflatex', 'drinker.tex'], stdout=subprocess.DEVNULL)
    #with open('backdrinker.tex', 'r') as f:
    #    assert(f.read() == document)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

from ..experiment import Experiment, RuntimeExperiment
from pprof.experiment import step, substep
from ..settings import config
from pprof import likwid
from pprof.utils.db import create_run, get_db_connection

from plumbum import local, FG
from plumbum.cmd import cp, awk, echo, tee, time, sed, rm
from os import path

polli = None
likwid_perfctr = None
pprof_calibrate = None
pprof_analyze = None
opt = None

import pdb

class RawRuntime(RuntimeExperiment):

    """ The polyjit experiment """

    def setup_commands(self):
        super(RawRuntime, self).setup_commands()
        global polli, likwid_perfctr, pprof_calibrate, pprof_analyze, opt
        bin_path = path.join(config["llvmdir"], "bin")
        likwid_path = path.join(config["likwiddir"], "bin")

        likwid_perfctr = local[path.join(likwid_path, "likwid-perfctr")]
        polli = local[path.join(bin_path, "polli")]
        pprof_calibrate = local[path.join(bin_path, "pprof-calibrate")]
        pprof_analyze = local[path.join(bin_path, "pprof-analyze")]
        opt = local[path.join(bin_path, "opt")]

    def run_project(self, p):
        from plumbum.cmd import time

        llvm_libs = path.join(config["llvmdir"], "lib")

        with step("RAW -O3"):
            p.ldflags = ["-L" + llvm_libs]
            p.cflags = ["-O3"]
            with substep("reconf & rebuild"):
                p.download()
                p.configure()
                p.build()
            with substep("run {}".format(p.name)):
                def run_with_time(run_f, args, has_stdin = False):
                    from plumbum.cmd import time
                    from pprof.utils.db import submit
                    run_cmd = time["-f", "%U-%S-%e", run_f]
                    if has_stdin:
                        run_cmd = ( run_cmd[args] < sys.stdin )
                    else:
                        run_cmd = run_cmd[args]
                    retcode, stdou, stderr = run_cmd.run()
                    run_id = create_run(
                        get_db_connection(), str(run_cmd), p.name, self.name, p.run_uuid)
                    timings = stderr.split('-')
                    timings = {
                        "table": "metrics",
                        "columns": ["name", "value", "run_id"],
                        "values": [
                            ("raw.time.user_s", timings[0], run_id),
                            ("raw.time.system_s", timings[1], run_id),
                            ("raw.time.real_s", timings[2], run_id)
                        ]
                    }
                    submit(timings)
                p.run(run_with_time)
<commit_msg>raw: fix typo<commit_after>#!/usr/bin/env python
# encoding: utf-8

from ..experiment import Experiment, RuntimeExperiment
from pprof.experiment import step, substep
from ..settings import config
from pprof import likwid
from pprof.utils.db import create_run, get_db_connection

from plumbum import local, FG
from plumbum.cmd import cp, awk, echo, tee, time, sed, rm
from os import path

polli = None
likwid_perfctr = None
pprof_calibrate = None
pprof_analyze = None
opt = None

import pdb

class RawRuntime(RuntimeExperiment):

    """ The polyjit experiment """

    def setup_commands(self):
        super(RawRuntime, self).setup_commands()
        global polli, likwid_perfctr, pprof_calibrate, pprof_analyze, opt
        bin_path = path.join(config["llvmdir"], "bin")
        likwid_path = path.join(config["likwiddir"], "bin")

        likwid_perfctr = local[path.join(likwid_path, "likwid-perfctr")]
        polli = local[path.join(bin_path, "polli")]
        pprof_calibrate = local[path.join(bin_path, "pprof-calibrate")]
        pprof_analyze = local[path.join(bin_path, "pprof-analyze")]
        opt = local[path.join(bin_path, "opt")]

    def run_project(self, p):
        from plumbum.cmd import time

        llvm_libs = path.join(config["llvmdir"], "lib")

        with step("RAW -O3"):
            p.ldflags = ["-L" + llvm_libs]
            p.cflags = ["-O3"]
            with substep("reconf & rebuild"):
                p.download()
                p.configure()
                p.build()
            with substep("run {}".format(p.name)):
                def run_with_time(run_f, args, has_stdin = False):
                    from plumbum.cmd import time
                    from pprof.utils.db import submit
                    run_cmd = time["-f", "%U-%S-%e", run_f]
                    if has_stdin:
                        run_cmd = ( run_cmd[args] < sys.stdin )
                    else:
                        run_cmd = run_cmd[args]
                    retcode, stdout, stderr = run_cmd.run()
                    run_id = create_run(
                        get_db_connection(), str(run_cmd), p.name, self.name, p.run_uuid)
                    timings = stderr.split('-')
                    timings = {
                        "table": "metrics",
                        "columns": ["name", "value", "run_id"],
                        "values": [
                            ("raw.time.user_s", timings[0], run_id),
                            ("raw.time.system_s", timings[1], run_id),
                            ("raw.time.real_s", timings[2], run_id)
                        ]
                    }
                    submit(timings)
                p.run(run_with_time)
<|endoftext|>


<commit_before>"""Support for media browsing."""
import logging

from homeassistant.components.media_player import BrowseError, BrowseMedia
from homeassistant.components.media_player.const import (
    MEDIA_CLASS_ALBUM,
    MEDIA_CLASS_ARTIST,
    MEDIA_CLASS_DIRECTORY,
    MEDIA_CLASS_EPISODE,
    MEDIA_CLASS_MOVIE,
    MEDIA_CLASS_MUSIC,
    MEDIA_CLASS_PLAYLIST,
    MEDIA_CLASS_SEASON,
    MEDIA_CLASS_TRACK,
    MEDIA_CLASS_TV_SHOW,
    MEDIA_TYPE_ALBUM,
    MEDIA_TYPE_ARTIST,
    MEDIA_TYPE_EPISODE,
    MEDIA_TYPE_MOVIE,
    MEDIA_TYPE_PLAYLIST,
    MEDIA_TYPE_SEASON,
    MEDIA_TYPE_TRACK,
    MEDIA_TYPE_TVSHOW,
)

PLAYABLE_MEDIA_TYPES = [
    MEDIA_TYPE_ALBUM,
    MEDIA_TYPE_ARTIST,
    MEDIA_TYPE_TRACK,
]

CONTAINER_TYPES_SPECIFIC_MEDIA_CLASS = {
    MEDIA_TYPE_ALBUM: MEDIA_CLASS_ALBUM,
    MEDIA_TYPE_ARTIST: MEDIA_CLASS_ARTIST,
    MEDIA_TYPE_PLAYLIST: MEDIA_CLASS_PLAYLIST,
    MEDIA_TYPE_SEASON: MEDIA_CLASS_SEASON,
    MEDIA_TYPE_TVSHOW: MEDIA_CLASS_TV_SHOW,
}

CHILD_TYPE_MEDIA_CLASS = {
    MEDIA_TYPE_SEASON: MEDIA_CLASS_SEASON,
    MEDIA_TYPE_ALBUM: MEDIA_CLASS_ALBUM,
    MEDIA_TYPE_ARTIST: MEDIA_CLASS_ARTIST,
    MEDIA_TYPE_MOVIE: MEDIA_CLASS_MOVIE,
    MEDIA_TYPE_PLAYLIST: MEDIA_CLASS_PLAYLIST,
    MEDIA_TYPE_TRACK: MEDIA_CLASS_TRACK,
    MEDIA_TYPE_TVSHOW: MEDIA_CLASS_TV_SHOW,
    MEDIA_TYPE_EPISODE: MEDIA_CLASS_EPISODE,
}

_LOGGER = logging.getLogger(__name__)

class UnknownMediaType(BrowseError):
    """Unknown media type."""

async def build_item_response(media_library, payload):
    """Create response payload for the provided media query."""
    search_id = payload["search_id"]
    search_type = payload["search_type"]

    thumbnail = None
    title = None
    media = None

    query = {"properties": ["thumbnail"]}
    # pylint: disable=protected-access
    if search_type == MEDIA_TYPE_ALBUM:
        if search_id:
            query.update({"filter": {"albumid": int(search_id)}})
            query["properties"].extend(
                ["albumid", "artist", "duration", "album", "track"]
            )
            album = await media_library._server.AudioLibrary.GetAlbumDetails(
                {"albumid": int(search_id), "properties": ["thumbnail"]}
            )
            thumbnail = media_library.thumbnail_url(
                album["albumdetails"].get("thumbnail")
            )
            title = album["albumdetails"]["label"]
            media = await media_library._server.AudioLibrary.GetSongs(query)
            media = media.get("songs")
        else:
            media = await media_library._server.AudioLibrary.GetAlbums(query)
            media = media.get("albums")
            title = "Albums"
    elif search_type == MEDIA_TYPE_ARTIST:
        if search_id:
            query.update({"filter": {"artistid": int(search_id)}})
            media = await media_library._server.AudioLibrary.GetAlbums(query)
            media = media.get("albums")
            artist = await media_library._server.AudioLibrary.GetArtistDetails(
                {"artistid": int(search_id), "properties": ["thumbnail"]}
            )
            thumbnail = media_library.thumbnail_url(
                artist["artistdetails"].get("thumbnail")
            )
            title = artist["artistdetails"]["label"]
        else:
            media = await media_library._server.AudioLibrary.GetArtists(query)
            media = media.get("artists")
            title = "Artists"
    elif search_type == "library_music":
        library = {MEDIA_TYPE_ALBUM: "Albums", MEDIA_TYPE_ARTIST: "Artists"}
        media = [{"label": name, "type": type_} for type_, name in library.items()]
        title = "Music Library"
    elif search_type == MEDIA_TYPE_MOVIE:
        media = await media_library._server.VideoLibrary.GetMovies(query)
        media = media.get("movies")
        title = "Movies"
    elif search_type == MEDIA_TYPE_TVSHOW:
        if search_id:
            media = await media_library._server.VideoLibrary.GetSeasons(
                {
                    "tvshowid": int(search_id),
                    "properties": ["thumbnail", "season", "tvshowid"],
                }
            )
            media = media.get("seasons")
            tvshow = await media_library._server.VideoLibrary.GetTVShowDetails(
                {"tvshowid": int(search_id), "properties": ["thumbnail"]}
            )
            thumbnail = media_library.thumbnail_url(
                tvshow["tvshowdetails"].get("thumbnail")
            )
            title = tvshow["tvshowdetails"]["label"]
        else:
            media = await media_library._server.VideoLibrary.GetTVShows(query)
            media = media.get("tvshows")
            title = "TV Shows"
    elif search_type == MEDIA_TYPE_SEASON:
        tv_show_id, season_id = search_id.split("/", 1)
        media = await media_library._server.VideoLibrary.GetEpisodes(
            {
                "tvshowid": int(tv_show_id),
                "season": int(season_id),
                "properties": ["thumbnail", "tvshowid", "seasonid"],
            }
        )
        media = media.get("episodes")
        if media:
            season = await media_library._server.VideoLibrary.GetSeasonDetails(
                {"seasonid": int(media[0]["seasonid"]), "properties": ["thumbnail"]}
            )
            thumbnail = media_library.thumbnail_url(
                season["seasondetails"].get("thumbnail")
            )
            title = season["seasondetails"]["label"]

    if media is None:
        return None

    children = []
    for item in media:
        try:
            children.append(item_payload(item, media_library))
        except UnknownMediaType:
            pass

    response = BrowseMedia(
        media_class=CONTAINER_TYPES_SPECIFIC_MEDIA_CLASS.get(
            search_type, MEDIA_CLASS_DIRECTORY
        ),
        media_content_id=search_id,
        media_content_type=search_type,
        title=title,
        can_play=search_type in PLAYABLE_MEDIA_TYPES and search_id,
        can_expand=True,
        children=children,
        thumbnail=thumbnail,
    )

    if search_type == "library_music":
        response.children_media_class = MEDIA_CLASS_MUSIC
    else:
        response.calculate_children_class()

    return response

def item_payload(item, media_library):
    """
    Create response payload for a single media item.

    Used by async_browse_media.
    """
    title = item["label"]
    thumbnail = item.get("thumbnail")
    if thumbnail:
        thumbnail = media_library.thumbnail_url(thumbnail)

    media_class = None

    if "songid" in item:
        media_content_type = MEDIA_TYPE_TRACK
        media_content_id = f"{item['songid']}"
        can_play = True
        can_expand = False
    elif "albumid" in item:
        media_content_type = MEDIA_TYPE_ALBUM
        media_content_id = f"{item['albumid']}"
        can_play = True
        can_expand = True
    elif "artistid" in item:
        media_content_type = MEDIA_TYPE_ARTIST
        media_content_id = f"{item['artistid']}"
        can_play = True
        can_expand = True
    elif "movieid" in item:
        media_content_type = MEDIA_TYPE_MOVIE
        media_content_id = f"{item['movieid']}"
        can_play = True
        can_expand = False
    elif "episodeid" in item:
        media_content_type = MEDIA_TYPE_EPISODE
        media_content_id = f"{item['episodeid']}"
        can_play = True
        can_expand = False
    elif "seasonid" in item:
        media_content_type = MEDIA_TYPE_SEASON
        media_content_id = f"{item['tvshowid']}/{item['season']}"
        can_play = False
        can_expand = True
    elif "tvshowid" in item:
        media_content_type = MEDIA_TYPE_TVSHOW
        media_content_id = f"{item['tvshowid']}"
        can_play = False
        can_expand = True
    else:
        # this case is for the top folder of each type
        # possible content types: album, artist, movie, library_music, tvshow
        media_class = MEDIA_CLASS_DIRECTORY
        media_content_type = item["type"]
        media_content_id = ""
        can_play = False
        can_expand = True

    if media_class is None:
        try:
            media_class = CHILD_TYPE_MEDIA_CLASS[media_content_type]
        except KeyError as err:
            _LOGGER.debug("Unknown media type received: %s", media_content_type)
            raise UnknownMediaType from err

    return BrowseMedia(
        title=title,
        media_class=media_class,
        media_content_type=media_content_type,
        media_content_id=media_content_id,
        can_play=can_play,
        can_expand=can_expand,
        thumbnail=thumbnail,
    )

def library_payload(media_library):
    """
    Create response payload to describe contents of a specific library.

    Used by async_browse_media.
    """
    library_info = BrowseMedia(
        media_class=MEDIA_CLASS_DIRECTORY,
        media_content_id="library",
        media_content_type="library",
        title="Media Library",
        can_play=False,
        can_expand=True,
        children=[],
    )

    library = {
        "library_music": "Music",
        MEDIA_TYPE_MOVIE: "Movies",
        MEDIA_TYPE_TVSHOW: "TV shows",
    }
    for item in [{"label": name, "type": type_} for type_, name in library.items()]:
        library_info.children.append(
            item_payload(
                {"label": item["label"], "type": item["type"], "uri": item["type"]},
                media_library,
            )
        )

    return library_info
<commit_msg>Add kodi browse media for channels (#40277)<commit_after>"""Support for media browsing."""
import logging

from homeassistant.components.media_player import BrowseError, BrowseMedia
from homeassistant.components.media_player.const import (
    MEDIA_CLASS_ALBUM,
    MEDIA_CLASS_ARTIST,
    MEDIA_CLASS_CHANNEL,
    MEDIA_CLASS_DIRECTORY,
    MEDIA_CLASS_EPISODE,
    MEDIA_CLASS_MOVIE,
    MEDIA_CLASS_MUSIC,
    MEDIA_CLASS_PLAYLIST,
    MEDIA_CLASS_SEASON,
    MEDIA_CLASS_TRACK,
    MEDIA_CLASS_TV_SHOW,
    MEDIA_TYPE_ALBUM,
    MEDIA_TYPE_ARTIST,
    MEDIA_TYPE_CHANNEL,
    MEDIA_TYPE_EPISODE,
    MEDIA_TYPE_MOVIE,
    MEDIA_TYPE_PLAYLIST,
    MEDIA_TYPE_SEASON,
    MEDIA_TYPE_TRACK,
    MEDIA_TYPE_TVSHOW,
)

PLAYABLE_MEDIA_TYPES = [
    MEDIA_TYPE_ALBUM,
    MEDIA_TYPE_ARTIST,
    MEDIA_TYPE_TRACK,
]

CONTAINER_TYPES_SPECIFIC_MEDIA_CLASS = {
    MEDIA_TYPE_ALBUM: MEDIA_CLASS_ALBUM,
    MEDIA_TYPE_ARTIST: MEDIA_CLASS_ARTIST,
    MEDIA_TYPE_PLAYLIST: MEDIA_CLASS_PLAYLIST,
    MEDIA_TYPE_SEASON: MEDIA_CLASS_SEASON,
    MEDIA_TYPE_TVSHOW: MEDIA_CLASS_TV_SHOW,
}

CHILD_TYPE_MEDIA_CLASS = {
    MEDIA_TYPE_SEASON: MEDIA_CLASS_SEASON,
    MEDIA_TYPE_ALBUM: MEDIA_CLASS_ALBUM,
    MEDIA_TYPE_ARTIST: MEDIA_CLASS_ARTIST,
    MEDIA_TYPE_MOVIE: MEDIA_CLASS_MOVIE,
    MEDIA_TYPE_PLAYLIST: MEDIA_CLASS_PLAYLIST,
    MEDIA_TYPE_TRACK: MEDIA_CLASS_TRACK,
    MEDIA_TYPE_TVSHOW: MEDIA_CLASS_TV_SHOW,
    MEDIA_TYPE_CHANNEL: MEDIA_CLASS_CHANNEL,
    MEDIA_TYPE_EPISODE: MEDIA_CLASS_EPISODE,
}

_LOGGER = logging.getLogger(__name__)

class UnknownMediaType(BrowseError):
    """Unknown media type."""

async def build_item_response(media_library, payload):
    """Create response payload for the provided media query."""
    search_id = payload["search_id"]
    search_type = payload["search_type"]

    thumbnail = None
    title = None
    media = None

    query = {"properties": ["thumbnail"]}
    # pylint: disable=protected-access
    if search_type == MEDIA_TYPE_ALBUM:
        if search_id:
            query.update({"filter": {"albumid": int(search_id)}})
            query["properties"].extend(
                ["albumid", "artist", "duration", "album", "track"]
            )
            album = await media_library._server.AudioLibrary.GetAlbumDetails(
                {"albumid": int(search_id), "properties": ["thumbnail"]}
            )
            thumbnail = media_library.thumbnail_url(
                album["albumdetails"].get("thumbnail")
            )
            title = album["albumdetails"]["label"]
            media = await media_library._server.AudioLibrary.GetSongs(query)
            media = media.get("songs")
        else:
            media = await media_library._server.AudioLibrary.GetAlbums(query)
            media = media.get("albums")
            title = "Albums"
    elif search_type == MEDIA_TYPE_ARTIST:
        if search_id:
            query.update({"filter": {"artistid": int(search_id)}})
            media = await media_library._server.AudioLibrary.GetAlbums(query)
            media = media.get("albums")
            artist = await media_library._server.AudioLibrary.GetArtistDetails(
                {"artistid": int(search_id), "properties": ["thumbnail"]}
            )
            thumbnail = media_library.thumbnail_url(
                artist["artistdetails"].get("thumbnail")
            )
            title = artist["artistdetails"]["label"]
        else:
            media = await media_library._server.AudioLibrary.GetArtists(query)
            media = media.get("artists")
            title = "Artists"
    elif search_type == "library_music":
        library = {MEDIA_TYPE_ALBUM: "Albums", MEDIA_TYPE_ARTIST: "Artists"}
        media = [{"label": name, "type": type_} for type_, name in library.items()]
        title = "Music Library"
    elif search_type == MEDIA_TYPE_MOVIE:
        media = await media_library._server.VideoLibrary.GetMovies(query)
        media = media.get("movies")
        title = "Movies"
    elif search_type == MEDIA_TYPE_TVSHOW:
        if search_id:
            media = await media_library._server.VideoLibrary.GetSeasons(
                {
                    "tvshowid": int(search_id),
                    "properties": ["thumbnail", "season", "tvshowid"],
                }
            )
            media = media.get("seasons")
            tvshow = await media_library._server.VideoLibrary.GetTVShowDetails(
                {"tvshowid": int(search_id), "properties": ["thumbnail"]}
            )
            thumbnail = media_library.thumbnail_url(
                tvshow["tvshowdetails"].get("thumbnail")
            )
            title = tvshow["tvshowdetails"]["label"]
        else:
            media = await media_library._server.VideoLibrary.GetTVShows(query)
            media = media.get("tvshows")
            title = "TV Shows"
    elif search_type == MEDIA_TYPE_SEASON:
        tv_show_id, season_id = search_id.split("/", 1)
        media = await media_library._server.VideoLibrary.GetEpisodes(
            {
                "tvshowid": int(tv_show_id),
                "season": int(season_id),
                "properties": ["thumbnail", "tvshowid", "seasonid"],
            }
        )
        media = media.get("episodes")
        if media:
            season = await media_library._server.VideoLibrary.GetSeasonDetails(
                {"seasonid": int(media[0]["seasonid"]), "properties": ["thumbnail"]}
            )
            thumbnail = media_library.thumbnail_url(
                season["seasondetails"].get("thumbnail")
            )
            title = season["seasondetails"]["label"]
    elif search_type == MEDIA_TYPE_CHANNEL:
        media = await media_library._server.PVR.GetChannels(
            {
                "channelgroupid": "alltv",
                "properties": ["thumbnail", "channeltype", "channel", "broadcastnow"],
            }
        )
        media = media.get("channels")
        title = "Channels"

    if media is None:
        return None

    children = []
    for item in media:
        try:
            children.append(item_payload(item, media_library))
        except UnknownMediaType:
            pass

    response = BrowseMedia(
        media_class=CONTAINER_TYPES_SPECIFIC_MEDIA_CLASS.get(
            search_type, MEDIA_CLASS_DIRECTORY
        ),
        media_content_id=search_id,
        media_content_type=search_type,
        title=title,
        can_play=search_type in PLAYABLE_MEDIA_TYPES and search_id,
        can_expand=True,
        children=children,
        thumbnail=thumbnail,
    )

    if search_type == "library_music":
        response.children_media_class = MEDIA_CLASS_MUSIC
    else:
        response.calculate_children_class()

    return response

def item_payload(item, media_library):
    """
    Create response payload for a single media item.

    Used by async_browse_media.
    """
    title = item["label"]
    thumbnail = item.get("thumbnail")
    if thumbnail:
        thumbnail = media_library.thumbnail_url(thumbnail)

    media_class = None

    if "songid" in item:
        media_content_type = MEDIA_TYPE_TRACK
        media_content_id = f"{item['songid']}"
        can_play = True
        can_expand = False
    elif "albumid" in item:
        media_content_type = MEDIA_TYPE_ALBUM
        media_content_id = f"{item['albumid']}"
        can_play = True
        can_expand = True
    elif "artistid" in item:
        media_content_type = MEDIA_TYPE_ARTIST
        media_content_id = f"{item['artistid']}"
        can_play = True
        can_expand = True
    elif "movieid" in item:
        media_content_type = MEDIA_TYPE_MOVIE
        media_content_id = f"{item['movieid']}"
        can_play = True
        can_expand = False
    elif "episodeid" in item:
        media_content_type = MEDIA_TYPE_EPISODE
        media_content_id = f"{item['episodeid']}"
        can_play = True
        can_expand = False
    elif "seasonid" in item:
        media_content_type = MEDIA_TYPE_SEASON
        media_content_id = f"{item['tvshowid']}/{item['season']}"
        can_play = False
        can_expand = True
    elif "tvshowid" in item:
        media_content_type = MEDIA_TYPE_TVSHOW
        media_content_id = f"{item['tvshowid']}"
        can_play = False
        can_expand = True
    elif "channelid" in item:
        media_content_type = MEDIA_TYPE_CHANNEL
        media_content_id = f"{item['channelid']}"
        broadcasting = item.get("broadcastnow")
        if broadcasting:
            show = broadcasting.get("title")
            title = f"{title} - {show}"
        can_play = True
        can_expand = False
    else:
        # this case is for the top folder of each type
        # possible content types: album, artist, movie, library_music, tvshow, channel
        media_class = MEDIA_CLASS_DIRECTORY
        media_content_type = item["type"]
        media_content_id = ""
        can_play = False
        can_expand = True

    if media_class is None:
        try:
            media_class = CHILD_TYPE_MEDIA_CLASS[media_content_type]
        except KeyError as err:
            _LOGGER.debug("Unknown media type received: %s", media_content_type)
            raise UnknownMediaType from err

    return BrowseMedia(
        title=title,
        media_class=media_class,
        media_content_type=media_content_type,
        media_content_id=media_content_id,
        can_play=can_play,
        can_expand=can_expand,
        thumbnail=thumbnail,
    )

def library_payload(media_library):
    """
    Create response payload to describe contents of a specific library.

    Used by async_browse_media.
    """
    library_info = BrowseMedia(
        media_class=MEDIA_CLASS_DIRECTORY,
        media_content_id="library",
        media_content_type="library",
        title="Media Library",
        can_play=False,
        can_expand=True,
        children=[],
    )

    library = {
        "library_music": "Music",
        MEDIA_TYPE_MOVIE: "Movies",
        MEDIA_TYPE_TVSHOW: "TV shows",
        MEDIA_TYPE_CHANNEL: "Channels",
    }
    for item in [{"label": name, "type": type_} for type_, name in library.items()]:
        library_info.children.append(
            item_payload(
                {"label": item["label"], "type": item["type"], "uri": item["type"]},
                media_library,
            )
        )

    return library_info
<|endoftext|>


<commit_before>import sqlalchemy as sa

json = None
try:
    import anyjson as json
except ImportError:
    import json as json

import six
from sqlalchemy.dialects.postgresql.base import ischema_names
from ..exceptions import ImproperlyConfigured

try:
    from sqlalchemy.dialects.postgresql import JSON
    has_postgres_json = True
except ImportError:
    class PostgresJSONType(sa.types.UserDefinedType):
        """
        Text search vector type for postgresql.
        """
        def get_col_spec(self):
            return 'json'

    ischema_names['json'] = PostgresJSONType
    has_postgres_json = False

class JSONType(sa.types.TypeDecorator):
    """
    JSONType offers way of saving JSON data structures to database. On
    PostgreSQL the underlying implementation of this data type is 'json' while
    on other databases its simply 'text'.

    ::

        from sqlalchemy_utils import JSONType

        class Product(Base):
            __tablename__ = 'product'
            id = sa.Column(sa.Integer, autoincrement=True)
            name = sa.Column(sa.Unicode(50))
            details = sa.Column(JSONType)

        product = Product()
        product.details = {
            'color': 'red',
            'type': 'car',
            'max-speed': '400 mph'
        }
        session.commit()
    """
    impl = sa.UnicodeText

    def __init__(self, *args, **kwargs):
        if json is None:
            raise ImproperlyConfigured(
                'JSONType needs anyjson package installed.'
            )
        super(JSONType, self).__init__(*args, **kwargs)

    def load_dialect_impl(self, dialect):
        if dialect.name == 'postgresql':
            # Use the native JSON type.
            if has_postgres_json:
                return dialect.type_descriptor(JSON())
            else:
                return dialect.type_descriptor(PostgresJSONType())
        else:
            return dialect.type_descriptor(self.impl)

    def process_bind_param(self, value, dialect):
        if dialect.name == 'postgresql' and has_postgres_json:
            return value
        if value is not None:
            value = six.text_type(json.dumps(value))
        return value

    def process_result_value(self, value, dialect):
        if dialect.name == 'postgresql':
            return value
        if value is not None:
            value = json.loads(value)
        return value
<commit_msg>Add absolute import, fixes #79<commit_after>from __future__ import absolute_import
import sqlalchemy as sa

json = None
try:
    import anyjson as json
except ImportError:
    import json as json

import six
from sqlalchemy.dialects.postgresql.base import ischema_names
from ..exceptions import ImproperlyConfigured

try:
    from sqlalchemy.dialects.postgresql import JSON
    has_postgres_json = True
except ImportError:
    class PostgresJSONType(sa.types.UserDefinedType):
        """
        Text search vector type for postgresql.
        """
        def get_col_spec(self):
            return 'json'

    ischema_names['json'] = PostgresJSONType
    has_postgres_json = False

class JSONType(sa.types.TypeDecorator):
    """
    JSONType offers way of saving JSON data structures to database. On
    PostgreSQL the underlying implementation of this data type is 'json' while
    on other databases its simply 'text'.

    ::

        from sqlalchemy_utils import JSONType

        class Product(Base):
            __tablename__ = 'product'
            id = sa.Column(sa.Integer, autoincrement=True)
            name = sa.Column(sa.Unicode(50))
            details = sa.Column(JSONType)

        product = Product()
        product.details = {
            'color': 'red',
            'type': 'car',
            'max-speed': '400 mph'
        }
        session.commit()
    """
    impl = sa.UnicodeText

    def __init__(self, *args, **kwargs):
        if json is None:
            raise ImproperlyConfigured(
                'JSONType needs anyjson package installed.'
            )
        super(JSONType, self).__init__(*args, **kwargs)

    def load_dialect_impl(self, dialect):
        if dialect.name == 'postgresql':
            # Use the native JSON type.
            if has_postgres_json:
                return dialect.type_descriptor(JSON())
            else:
                return dialect.type_descriptor(PostgresJSONType())
        else:
            return dialect.type_descriptor(self.impl)

    def process_bind_param(self, value, dialect):
        if dialect.name == 'postgresql' and has_postgres_json:
            return value
        if value is not None:
            value = six.text_type(json.dumps(value))
        return value

    def process_result_value(self, value, dialect):
        if dialect.name == 'postgresql':
            return value
        if value is not None:
            value = json.loads(value)
        return value
<|endoftext|>


<commit_before># Day 0: Mean, Median, and Mode
# https://www.hackerrank.com/challenges/s10-basic-statistics

from __future__ import division

# TODO: Fix inputs
def get_input_elements():
    number_of_items = input()
    elements = []
    for index in range(number_of_items):
        elements.append(input())
    return elements

def get_mean(elements):
    """The average of all the integers in a set of values."""
    s = 0
    for element in elements:
        s += element
    return (s / len(elements))

def get_median(elements):
    """The midpoint value of a data set for which an equal number of samples are
    less than and greater than the value. For an odd sample size, this is the
    middle element of the sorted sample; for an even sample size, this is the
    average of the  middle elements of the sorted sample."""
    elements.sort()
    length_of_elements = len(elements)
    mid_position = length_of_elements // 2
    if length_of_elements % 2 == 0:
        # for even number of items, the median is the average of the 2 middle
        # elements
        sum_of_the_mids = (elements[mid_position - 1] + elements[mid_position])
        median = sum_of_the_mids / 2
    else:
        # for odd number of items, the median is the middle element
        median = elements[mid_position]
    return median

def get_mode(elements):
    """The element(s) that occur most frequently in a data set."""
    dictionary = {}
    elements.sort()
    for element in elements:
        if element in dictionary:
            dictionary[element] += 1
        else:
            dictionary[element] = 1

    mode = elements[0]
    highest_count = 0
    for key, value in dictionary.items():
        if highest_count <= value and mode > key:
            mode = key
            highest_count = value
    return mode

elements = get_input_elements()
mean = get_mean(elements)
median = get_median(elements)
mode = get_mode(elements)

print(mean)
print(median)
print(mode)
<commit_msg>Fix issue with getting the inputs<commit_after># Day 0: Mean, Median, and Mode
# https://www.hackerrank.com/challenges/s10-basic-statistics

from __future__ import division

def get_input_elements():
    number_of_items = raw_input()
    if number_of_items < 10 and number_of_items > 2500:
        sys.exit('The number of items is < 10 or > 2500')
    list_of_numbers = raw_input()
    elements = list_of_numbers.split(' ')
    return [int(element) for element in elements]

def get_mean(elements):
    """The average of all the integers in a set of values."""
    s = 0
    for element in elements:
        s += element
    return (s / len(elements))

def get_median(elements):
    """The midpoint value of a data set for which an equal number of samples are
    less than and greater than the value. For an odd sample size, this is the
    middle element of the sorted sample; for an even sample size, this is the
    average of the  middle elements of the sorted sample."""
    elements.sort()
    length_of_elements = len(elements)
    mid_position = length_of_elements // 2
    if length_of_elements % 2 == 0:
        # for even number of items, the median is the average of the 2 middle
        # elements
        sum_of_the_mids = (elements[mid_position - 1] + elements[mid_position])
        median = sum_of_the_mids / 2
    else:
        # for odd number of items, the median is the middle element
        median = elements[mid_position]
    return median

def get_mode(elements):
    """The element(s) that occur most frequently in a data set."""
    dictionary = {}
    elements.sort()
    for element in elements:
        if element in dictionary:
            dictionary[element] += 1
        else:
            dictionary[element] = 1

    mode = elements[0]
    highest_count = 0
    for key, value in dictionary.items():
        if highest_count <= value and mode > key:
            mode = key
            highest_count = value
    return mode

elements = get_input_elements()
mean = get_mean(elements)
median = get_median(elements)
mode = get_mode(elements)

print(round(mean, 1))
print(round(median, 1))
print(mode)
<|endoftext|>


<commit_before>#!/usr/bin/python3

import sys
import glob
import subprocess
import re
import collections
import os.path
import shlex

def str2bool(s):
    return {
        "true": True,
        "false": False
    }[s]

def extract_bool(p, s):
    m = re.search(p, s)
    return str2bool(m.group(1)) if m else None

set_filename = sys.argv[1]
with open(set_filename) as set_file:
    stats = collections.defaultdict(int)

    code_filenames = []
    for pattern in set_file:
        pattern = pattern.strip()
        pattern = os.path.join(os.path.dirname(set_filename), pattern)
        if pattern:
            for code_filename in glob.iglob(pattern):
                code_filenames.append(code_filename)

    for code_filename in sorted(code_filenames):
        print(f"{code_filename}: ", end="", flush=True)
        expected = extract_bool(r"_(false|true)-unreach-call", code_filename)

        try:
            # p = subprocess.run(f"~/Desktop/sv-comp/goblint/goblint --enable ana.sv-comp --enable dbg.debug {code_filename}", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8")
            # p = subprocess.run(f"~/Desktop/sv-comp/goblint/goblint --enable ana.sv-comp --enable dbg.debug --set ana.activated[+] \"'var_eq'\" --enable ana.int.interval {code_filename}", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8")
            # p = subprocess.run(shlex.split(f"/home/simmo/Desktop/sv-comp/goblint/goblint --enable ana.sv-comp --enable dbg.debug --set ana.activated[+] \"'var_eq'\" --enable ana.int.interval {code_filename}"), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8", timeout=10)
            p = subprocess.run(shlex.split(f"/home/simmo/Desktop/sv-comp/goblint/goblint --enable ana.sv-comp --enable dbg.debug --set ana.activated[+] \"'var_eq'\" --set ana.activated[+] \"'symb_locks'\" --set ana.activated[+] \"'thread'\" --set ana.activated[+] \"'region'\" --enable ana.int.interval {code_filename}"), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8", timeout=30)
            actual = extract_bool(r"SV-COMP \(unreach-call\): (false|true)", p.stdout)
        except subprocess.TimeoutExpired:
            actual = "timeout"

        if "p" in locals(): # sometimes on timeout p is declared, sometimes isn't
            missing_funcs = False
            for m in re.finditer(r"Function definition missing for (__VERIFIER_\S+)", p.stdout):
                missing_funcs = True
                print(f"MISSING FUNC {m.group(1)}")

            if missing_funcs:
                sys.exit(0)

        text = None
        if expected is None or actual is None:
            text = f"NONE expected {expected}, actual {actual}"
        elif actual == expected:
            text = f"CORRECT {expected}"
        elif actual == "timeout":
            text = f"UNKNOWN expected {expected}, actual {actual}"
        else:
            text = f"INCORRECT expected {expected}, actual {actual}"

        print(text)
        stats[text] += 1

    print("-" * 80)
    for text, count in stats.items():
        print(f"{text}: {count}")<commit_msg>Add test counting (without running) to SV-COMP runner script<commit_after>#!/usr/bin/python3

import sys
import glob
import subprocess
import re
import collections
import os.path
import shlex

overview = True

def str2bool(s):
    return {
        "true": True,
        "false": False
    }[s]

def extract_bool(p, s):
    m = re.search(p, s)
    return str2bool(m.group(1)) if m else None

set_filename = sys.argv[1]
with open(set_filename) as set_file:
    stats = collections.defaultdict(int)

    code_filenames = []
    for pattern in set_file:
        pattern = pattern.strip()
        pattern = os.path.join(os.path.dirname(set_filename), pattern)
        if pattern:
            for code_filename in glob.iglob(pattern):
                code_filenames.append(code_filename)

    for code_filename in sorted(code_filenames):
        print(f"{code_filename}: ", end="", flush=True)
        expected = extract_bool(r"_(false|true)-unreach-call", code_filename)

        if overview:
            actual = None
        else:
            try:
                # p = subprocess.run(f"~/Desktop/sv-comp/goblint/goblint --enable ana.sv-comp --enable dbg.debug {code_filename}", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8")
                # p = subprocess.run(f"~/Desktop/sv-comp/goblint/goblint --enable ana.sv-comp --enable dbg.debug --set ana.activated[+] \"'var_eq'\" --enable ana.int.interval {code_filename}", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8")
                # p = subprocess.run(shlex.split(f"/home/simmo/Desktop/sv-comp/goblint/goblint --enable ana.sv-comp --enable dbg.debug --set ana.activated[+] \"'var_eq'\" --enable ana.int.interval {code_filename}"), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8", timeout=10)
                p = subprocess.run(shlex.split(f"/home/simmo/Desktop/sv-comp/goblint/goblint --enable ana.sv-comp --enable dbg.debug --set ana.activated[+] \"'var_eq'\" --set ana.activated[+] \"'symb_locks'\" --set ana.activated[+] \"'thread'\" --set ana.activated[+] \"'region'\" --enable ana.int.interval {code_filename}"), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8", timeout=30)
                actual = extract_bool(r"SV-COMP \(unreach-call\): (false|true)", p.stdout)
            except subprocess.TimeoutExpired:
                actual = "timeout"

        if "p" in locals(): # sometimes on timeout p is declared, sometimes isn't
            missing_funcs = False
            for m in re.finditer(r"Function definition missing for (__VERIFIER_\S+)", p.stdout):
                missing_funcs = True
                print(f"MISSING FUNC {m.group(1)}")

            if missing_funcs:
                sys.exit(0)

        text = None
        if expected is None or actual is None:
            text = f"NONE expected {expected}, actual {actual}"
        elif actual == expected:
            text = f"CORRECT {expected}"
        elif actual == "timeout":
            text = f"UNKNOWN expected {expected}, actual {actual}"
        else:
            text = f"INCORRECT expected {expected}, actual {actual}"

        print(text)
        stats[text] += 1

    print("-" * 80)
    for text, count in stats.items():
        print(f"{text}: {count}")<|endoftext|>


<commit_before>import sys
from typing import Dict, Type

if sys.version_info >= (3, 8):
    from typing import TypedDict  # pylint: disable=no-name-in-module
else:
    from typing_extensions import TypedDict

from .base import Detector, RegexDetector
from .credential import CredentialDetector
from .credit_card import CreditCardDetector
from .drivers_licence import DriversLicenceDetector
from .email import EmailDetector
from .phone import PhoneDetector
from .postalcode import PostalCodeDetector
from .known import KnownFilthDetector
from .twitter import TwitterDetector
from .url import UrlDetector
from .vehicle_licence_plate import VehicleLicencePlateDetector
from .en_GB.nino import NINODetector
from .en_GB.trn import TrnDetector
from .en_US.social_security_number import SocialSecurityNumberDetector

DetectorConfigurationItem = TypedDict(
    'DetectorConfigurationItem',
    {'detector': Type[Detector], 'autoload': bool}
)

detector_configuration = {
    # Detectors that are automatically loaded by scrubadub
    CredentialDetector.name: {'detector': CredentialDetector, 'autoload': True},
    CreditCardDetector.name: {'detector': CreditCardDetector, 'autoload': True},
    DriversLicenceDetector.name: {'detector': DriversLicenceDetector, 'autoload': True},
    EmailDetector.name: {'detector': EmailDetector, 'autoload': True},
    NINODetector.name: {'detector': NINODetector, 'autoload': True},
    PhoneDetector.name: {'detector': PhoneDetector, 'autoload': True},
    SocialSecurityNumberDetector.name: {'detector': SocialSecurityNumberDetector, 'autoload': True},
    TrnDetector.name: {'detector': TrnDetector, 'autoload': True},
    TwitterDetector.name: {'detector': TwitterDetector, 'autoload': True},
    UrlDetector.name: {'detector': UrlDetector, 'autoload': True},
    VehicleLicencePlateDetector.name: {'detector': VehicleLicencePlateDetector, 'autoload': True},
    # Detectors that are not automatically loaded by scrubadub
    KnownFilthDetector.name: {'detector': KnownFilthDetector, 'autoload': False},
    PostalCodeDetector.name: {'detector': PostalCodeDetector, 'autoload': False},
}  # type: Dict[str, DetectorConfigurationItem]

def register_detector(detector: Type[Detector], autoload: bool = False):
    """Register a detector for use with the ``Scrubber`` class.

    This is used when you dont want to have to import a detector by default.
    It may be useful for certain detectors with large or unusal dependancies, which you may not always want to import.
    In this case you can use ``register_detector(NewDetector, autoload=True)`` after your detector definition so that
    if the file is imported it wil be automatically registered.
    This will mean that you don't need to import the ``NewDetector`` in this file and so it's dependencies won't need
    to be installed just to import this package.

    The argument ``autoload`` sets if a new ``Scrubber()`` instance should load this ``Detector`` by default.

    .. code:: pycon

        >>> import scrubadub
        >>> class NewDetector(scrubadub.detectors.Detector):
        ...     pass
        >>> scrubadub.detectors.register_detector(NewDetector, autoload=False)

    :param detector: The ``Detector`` to register with the scrubadub detector configuration.
    :type detector: Detector class
    :param autoload: Whether to automatically load this ``Detector`` on ``Scrubber`` initialisation.
    :type autoload: bool
    """
    detector_configuration[detector.name] = {
        'detector': detector,
        'autoload': autoload,
    }
    current_module = __import__(__name__)
    setattr(current_module.detectors, detector.__name__, detector)
<commit_msg>add a comment about the detector_configuration<commit_after>import sys
from typing import Dict, Type

if sys.version_info >= (3, 8):
    from typing import TypedDict  # pylint: disable=no-name-in-module
else:
    from typing_extensions import TypedDict

from .base import Detector, RegexDetector
from .credential import CredentialDetector
from .credit_card import CreditCardDetector
from .drivers_licence import DriversLicenceDetector
from .email import EmailDetector
from .phone import PhoneDetector
from .postalcode import PostalCodeDetector
from .known import KnownFilthDetector
from .twitter import TwitterDetector
from .url import UrlDetector
from .vehicle_licence_plate import VehicleLicencePlateDetector
from .en_GB.nino import NINODetector
from .en_GB.trn import TrnDetector
from .en_US.social_security_number import SocialSecurityNumberDetector

DetectorConfigurationItem = TypedDict(
    'DetectorConfigurationItem',
    {'detector': Type[Detector], 'autoload': bool}
)

# This dictonary is used in scrubadub.scrubber.Scrubber.__init__, to find available detecotrs that should be used to
# scrub dirty text. The autoload option says if the scrubber should automatically load this detector if no
# `detector_list` is provided to the `Scrubber()`.
detector_configuration = {
    # Detectors that are automatically loaded by scrubadub
    CredentialDetector.name: {'detector': CredentialDetector, 'autoload': True},
    CreditCardDetector.name: {'detector': CreditCardDetector, 'autoload': True},
    DriversLicenceDetector.name: {'detector': DriversLicenceDetector, 'autoload': True},
    EmailDetector.name: {'detector': EmailDetector, 'autoload': True},
    NINODetector.name: {'detector': NINODetector, 'autoload': True},
    PhoneDetector.name: {'detector': PhoneDetector, 'autoload': True},
    PostalCodeDetector.name: {'detector': PostalCodeDetector, 'autoload': True},
    SocialSecurityNumberDetector.name: {'detector': SocialSecurityNumberDetector, 'autoload': True},
    TrnDetector.name: {'detector': TrnDetector, 'autoload': True},
    TwitterDetector.name: {'detector': TwitterDetector, 'autoload': True},
    UrlDetector.name: {'detector': UrlDetector, 'autoload': True},
    VehicleLicencePlateDetector.name: {'detector': VehicleLicencePlateDetector, 'autoload': True},
    # Detectors that are not automatically loaded by scrubadub
    KnownFilthDetector.name: {'detector': KnownFilthDetector, 'autoload': False},
}  # type: Dict[str, DetectorConfigurationItem]

def register_detector(detector: Type[Detector], autoload: bool = False):
    """Register a detector for use with the ``Scrubber`` class.

    This is used when you dont want to have to import a detector by default.
    It may be useful for certain detectors with large or unusal dependancies, which you may not always want to import.
    In this case you can use ``register_detector(NewDetector, autoload=True)`` after your detector definition so that
    if the file is imported it wil be automatically registered.
    This will mean that you don't need to import the ``NewDetector`` in this file and so it's dependencies won't need
    to be installed just to import this package.

    The argument ``autoload`` sets if a new ``Scrubber()`` instance should load this ``Detector`` by default.

    .. code:: pycon

        >>> import scrubadub
        >>> class NewDetector(scrubadub.detectors.Detector):
        ...     pass
        >>> scrubadub.detectors.register_detector(NewDetector, autoload=False)

    :param detector: The ``Detector`` to register with the scrubadub detector configuration.
    :type detector: Detector class
    :param autoload: Whether to automatically load this ``Detector`` on ``Scrubber`` initialisation.
    :type autoload: bool
    """
    detector_configuration[detector.name] = {
        'detector': detector,
        'autoload': autoload,
    }
    current_module = __import__(__name__)
    setattr(current_module.detectors, detector.__name__, detector)
<|endoftext|>