<commit_before>import pandas as pd
import numpy as np
import krippendorff
from krippendorff import alpha
from data_normalization_functions import *

def nominal_metric(a, b):
    return a != b

def interval_metric(a, b):
    return (a-b)**2

def ratio_metric(a, b):
    return ((a-b)/(a+b))**2

def krippendorff_alpha(data, metric=interval_metric, force_vecmath=False, convert_items=float, missing_items=None):
    '''
    Calculate Krippendorff's alpha (inter-rater reliability):

    data is in the format
    [
        {unit1:value, unit2:value, ...},  # coder 1
        {unit1:value, unit3:value, ...},   # coder 2
        ...                            # more coders
    ]
    or
    it is a sequence of (masked) sequences (list, numpy.array, numpy.ma.array, e.g.) with rows corresponding to coders and
    columns to items

    metric: function calculating the pairwise distance
    force_vecmath: force vector math for custom metrics (numpy required)
    convert_items: function for the type conversion of items (default: float)
    missing_items: indicator for missing items (default: None)
    '''

    # number of coders
    m = len(data)

    # set of constants identifying missing values
    if missing_items is None:
        maskitems = []
    else:
        maskitems = list(missing_items)
    if np is not None:
        maskitems.append(np.ma.masked_singleton)

    # convert input data to a dict of items
    units = {}
    for d in data:
        try:
            # try if d behaves as a dict
            diter = d.items()
        except AttributeError:
            # sequence assumed for d
            diter = enumerate(d)

        for it, g in diter:
            if g not in maskitems:
                try:
                    its = units[it]
                except KeyError:
                    its = []
                    units[it] = its
                its.append(convert_items(g))

    units = dict((it, d) for it, d in units.items() if len(d) > 1)  # units with pairable values
    n = sum(len(pv) for pv in units.values())  # number of pairable values

    if n == 0:
        raise ValueError("No items to compare.")

    np_metric = (np is not None) and ((metric in (interval_metric, nominal_metric, ratio_metric)) or force_vecmath)

    Do = 0.
    for grades in units.values():
        if np_metric:
            gr = np.asarray(grades)
            Du = sum(np.sum(metric(gr, gri)) for gri in gr)
        else:
            Du = sum(metric(gi, gj) for gi in grades for gj in grades)
        Do += Du/float(len(grades)-1)
    Do /= float(n)

    if Do == 0:
        return 1.

    De = 0.
    for g1 in units.values():
        if np_metric:
            d1 = np.asarray(g1)
            for g2 in units.values():
                De += sum(np.sum(metric(d1, gj)) for gj in g2)
        else:
            for g2 in units.values():
                De += sum(metric(gi, gj) for gi in g1 for gj in g2)
    De /= float(n*(n-1))

    return 1.-Do/De if (Do and De) else 1.

def calculate_inter_annotator_agreement(df, question_column, golden=False):
    '''
    :df: (pandas.DataFrame) the dataframe containing all crowd annotations
    :golden: (bool) a variable indicating whether we are measuring agreement
             on gold or non-gold questions
    :retuen: None
    Description:
    '''

    # If golden flag is False, get rid of the gold questions
    if golden == False:
        df = df.query('_golden == False')
    else:
        df = df.query('_golden == True')

    nannotators = len(df['_worker_id'].unique())
    ncols = len(df['comment_counter'].unique().tolist())

    # Create a new DataFrame for inter-annotator agreement data
    annotation_df = pd.DataFrame(np.full((nannotators, ncols), '*'),
                                 index = df['_worker_id'].unique().tolist(),
                                 columns = df['comment_counter'].unique().tolist())

    for index, row in df.iterrows():
        annotation_df.loc[row['_worker_id']][row['comment_counter']] = 1 if row[question_column] == 'yes' else 0

    reliability_data = annotation_df.values.tolist()
    print('Length of the data: ', len(reliability_data))

    missing = '*' # indicator for missing values

    k_alpha = krippendorff_alpha(reliability_data, metric = nominal_metric, missing_items=missing)
    print("Krippendorff's alpha with nominal metric: %.3f" % k_alpha)
    return k_alpha

def secret_gold_evaluation_constructiveness(df):
    '''
    :df: (pandas.DataFrame)
    :return: (float) Secret gold evaluation results for constructiveness
    '''
    # Get a subset dataframe with internal gold questions for constructiveness
    internal_gold_con_df = df[df['constructive_internal_gold'].notnull()].copy()
    percentage_multiplier = 100.0/internal_gold_con_df.shape[0]

    # Binarize constructiveness for easy comparison.
    internal_gold_con_df['constructive_binary'] = internal_gold_con_df['constructive'].apply(binarize_constructiveness)

    # Find out the number of instances of disagreement between the gold and the aggregated crowd answer.
    # If this number is high (>5, more than 25%) we should be worried.
    constructiveness_disagreement_df = internal_gold_con_df[internal_gold_con_df['constructive_binary'] !=
                                       internal_gold_con_df['constructive_internal_gold']]

    #show_comments(constructiveness_disagreement_df, ['constructive_internal_gold', 'constructive_binary'])
    return round(constructiveness_disagreement_df.shape[0] * percentage_multiplier, 2)

def secret_gold_evaluation_toxicity(df):
    '''
    :df: (pandas.DataFrame)
    :return: (float) Secret gold evaluation results for toxicity
    '''
    # Get a subset dataframe with internal gold questions for toxicity
    internal_gold_tox_df = df[df['crowd_toxicity_level_internal_gold'].notnull()].copy()
    percentage_multiplier = 100.0/internal_gold_tox_df.shape[0]
    internal_gold_tox_df['crowd_toxicity_level_binary'] = internal_gold_tox_df['crowd_toxicity_level'].apply(binarize_toxicity)
    internal_gold_tox_df['crowd_toxicity_level_internal_gold_binary'] = \
    (internal_gold_tox_df['crowd_toxicity_level_internal_gold'].apply(binarize_toxicity_with_multiple_answers))
    # Find out the number of instances of disagreement between the gold and the aggregated crowd answer.
    # If this number is high (>5, more than 25%) we should be worried.
    toxicity_disagreement_df = (internal_gold_tox_df[internal_gold_tox_df['crowd_toxicity_level_internal_gold_binary'] !=
                                                    internal_gold_tox_df['crowd_toxicity_level_binary']])

    #show_comments(toxicity_disagreement_df, ['crowd_toxicity_level_internal_gold_binary', 'crowd_toxicity_level_binary'] )
    return round(toxicity_disagreement_df.shape[0] * percentage_multiplier, 2)

if __name__ == "__main__":
    pass
    <commit_msg>deleting commented lines<commit_after>import pandas as pd
import numpy as np
import krippendorff
from krippendorff import alpha
from data_normalization_functions import *

def nominal_metric(a, b):
    return a != b

def interval_metric(a, b):
    return (a-b)**2

def ratio_metric(a, b):
    return ((a-b)/(a+b))**2

def krippendorff_alpha(data, metric=interval_metric, force_vecmath=False, convert_items=float, missing_items=None):
    '''
    Calculate Krippendorff's alpha (inter-rater reliability):

    data is in the format
    [
        {unit1:value, unit2:value, ...},  # coder 1
        {unit1:value, unit3:value, ...},   # coder 2
        ...                            # more coders
    ]
    or
    it is a sequence of (masked) sequences (list, numpy.array, numpy.ma.array, e.g.) with rows corresponding to coders and
    columns to items

    metric: function calculating the pairwise distance
    force_vecmath: force vector math for custom metrics (numpy required)
    convert_items: function for the type conversion of items (default: float)
    missing_items: indicator for missing items (default: None)
    '''

    # number of coders
    m = len(data)

    # set of constants identifying missing values
    if missing_items is None:
        maskitems = []
    else:
        maskitems = list(missing_items)
    if np is not None:
        maskitems.append(np.ma.masked_singleton)

    # convert input data to a dict of items
    units = {}
    for d in data:
        try:
            # try if d behaves as a dict
            diter = d.items()
        except AttributeError:
            # sequence assumed for d
            diter = enumerate(d)

        for it, g in diter:
            if g not in maskitems:
                try:
                    its = units[it]
                except KeyError:
                    its = []
                    units[it] = its
                its.append(convert_items(g))

    units = dict((it, d) for it, d in units.items() if len(d) > 1)  # units with pairable values
    n = sum(len(pv) for pv in units.values())  # number of pairable values

    if n == 0:
        raise ValueError("No items to compare.")

    np_metric = (np is not None) and ((metric in (interval_metric, nominal_metric, ratio_metric)) or force_vecmath)

    Do = 0.
    for grades in units.values():
        if np_metric:
            gr = np.asarray(grades)
            Du = sum(np.sum(metric(gr, gri)) for gri in gr)
        else:
            Du = sum(metric(gi, gj) for gi in grades for gj in grades)
        Do += Du/float(len(grades)-1)
    Do /= float(n)

    if Do == 0:
        return 1.

    De = 0.
    for g1 in units.values():
        if np_metric:
            d1 = np.asarray(g1)
            for g2 in units.values():
                De += sum(np.sum(metric(d1, gj)) for gj in g2)
        else:
            for g2 in units.values():
                De += sum(metric(gi, gj) for gi in g1 for gj in g2)
    De /= float(n*(n-1))

    return 1.-Do/De if (Do and De) else 1.

def calculate_inter_annotator_agreement(df, question_column, golden=False):
    '''
    :df: (pandas.DataFrame) the dataframe containing all crowd annotations
    :golden: (bool) a variable indicating whether we are measuring agreement
             on gold or non-gold questions
    :retuen: None
    Description:
    '''

    # If golden flag is False, get rid of the gold questions
    if golden == False:
        df = df.query('_golden == False')
    else:
        df = df.query('_golden == True')

    nannotators = len(df['_worker_id'].unique())
    ncols = len(df['comment_counter'].unique().tolist())

    # Create a new DataFrame for inter-annotator agreement data
    annotation_df = pd.DataFrame(np.full((nannotators, ncols), '*'),
                                 index = df['_worker_id'].unique().tolist(),
                                 columns = df['comment_counter'].unique().tolist())

    for index, row in df.iterrows():
        annotation_df.loc[row['_worker_id']][row['comment_counter']] = 1 if row[question_column] == 'yes' else 0

    reliability_data = annotation_df.values.tolist()
    print('Length of the data: ', len(reliability_data))

    missing = '*' # indicator for missing values

    k_alpha = krippendorff_alpha(reliability_data, metric = nominal_metric, missing_items=missing)
    print("Krippendorff's alpha with nominal metric: %.3f" % k_alpha)
    return k_alpha

def secret_gold_evaluation_constructiveness(df):
    '''
    :df: (pandas.DataFrame)
    :return: (float) Secret gold evaluation results for constructiveness
    '''
    # Get a subset dataframe with internal gold questions for constructiveness
    internal_gold_con_df = df[df['constructive_internal_gold'].notnull()].copy()
    percentage_multiplier = 100.0/internal_gold_con_df.shape[0]

    # Binarize constructiveness for easy comparison.
    internal_gold_con_df['constructive_binary'] = internal_gold_con_df['constructive'].apply(binarize_constructiveness)

    # Find out the number of instances of disagreement between the gold and the aggregated crowd answer.
    # If this number is high (>5, more than 25%) we should be worried.
    constructiveness_disagreement_df = internal_gold_con_df[internal_gold_con_df['constructive_binary'] !=
                                       internal_gold_con_df['constructive_internal_gold']]

    #show_comments(constructiveness_disagreement_df, ['constructive_internal_gold', 'constructive_binary'])
    return round(constructiveness_disagreement_df.shape[0] * percentage_multiplier, 2)

def secret_gold_evaluation_toxicity(df):
    '''
    :df: (pandas.DataFrame)
    :return: (float) Secret gold evaluation results for toxicity
    '''
    # Get a subset dataframe with internal gold questions for toxicity
    internal_gold_tox_df = df[df['crowd_toxicity_level_internal_gold'].notnull()].copy()
    percentage_multiplier = 100.0/internal_gold_tox_df.shape[0]
    internal_gold_tox_df['crowd_toxicity_level_binary'] = internal_gold_tox_df['crowd_toxicity_level'].apply(binarize_toxicity)
    internal_gold_tox_df['crowd_toxicity_level_internal_gold_binary'] = \
    (internal_gold_tox_df['crowd_toxicity_level_internal_gold'].apply(binarize_toxicity_with_multiple_answers))
    # Find out the number of instances of disagreement between the gold and the aggregated crowd answer.
    # If this number is high (>5, more than 25%) we should be worried.
    toxicity_disagreement_df = (internal_gold_tox_df[internal_gold_tox_df['crowd_toxicity_level_internal_gold_binary'] !=
                                                    internal_gold_tox_df['crowd_toxicity_level_binary']])

    return round(toxicity_disagreement_df.shape[0] * percentage_multiplier, 2)

if __name__ == "__main__":
    pass
    <|endoftext|>


<commit_before>#!/usr/bin/env python
# coding=utf-8

"""
ROS node for publishing "/imu/data" and "odom" (nav_msgs/Odometry) topics,
using the Adafruit BNO055 9DOF board.

Author:  Markus Knapp, 2017
Website: https://direcs.de
"""
#
# Adafruit BNO055 code:
#
# Copyright (c) 2015 Adafruit Industries
# Original-Author: Tony DiCola
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
import logging
import sys
import time

import rospy
import math

from std_msgs.msg import Header, Float32
from sensor_msgs.msg import Imu, Temperature

# see also http://wiki.ros.org/navigation/Tutorials/RobotSetup/Odom:
# "The nav_msgs/Odometry message stores an estimate of the position
#  and velocity of a robot in free space"
from nav_msgs.msg import Odometry
# for the tf broadcaster
import tf

# sleep time for this node in seconds
sleepTime = 0.25

# for getting the hostname of the underlying system
import socket
# showing hostname
hostname = socket.gethostname()
rospy.loginfo("Running on host %s.", hostname)
if hostname != 'minibot':
    rospy.logwarn("Test mode only due to other host. Skipping all I2C staff!")

# the odometry topic
pubOdom = rospy.Publisher('odom', Odometry, queue_size=50)
# the tf broadcaster
# "any odometry source must publish information about the coordinate frame that it manages"
odomBroadcaster = tf.TransformBroadcaster() # "br" in tf_broadcaster.py

# Euler topics
pubH = rospy.Publisher('heading', Float32, queue_size=1)
pubR = rospy.Publisher('roll',    Float32, queue_size=1)
pubP = rospy.Publisher('pitch',   Float32, queue_size=1)
# Temperature
pubTemp = rospy.Publisher('temperature',   Temperature, queue_size=1)
# ROS IMU format messages
pubImu  = rospy.Publisher('imu/data', Imu, queue_size=1)

# header frame for odometry message
frame_id = 'odom' #
child_id = 'base_link' # normally the coordinate frame of the mobile base, so base_link.
seq = 0

# define temperature message
temp_msg = Temperature()
# ignore the covariance data, it is optional
temp_msg.variance = 0

# define IMU message
imu_msg = Imu()
# ignore the covariance data, it is optional
imu_msg.orientation_covariance[0]         = -1
imu_msg.angular_velocity_covariance[0]    = -1
imu_msg.linear_acceleration_covariance[0] = -1

# initialise the ROS node
rospy.init_node('bno055_node')

# run some parts only on the real robot
if hostname == 'minibot':
    # load library
    from Adafruit_BNO055 import BNO055

    # Create and configure the BNO sensor connection.
    # Using I2C without a RST pin
    bno = BNO055.BNO055()

    # Initialize the BNO055 and stop if something went wrong.
    if not bno.begin():
        rospy.logerr("Failed to initialize BNO055! Is the sensor connected?")

    # Print system status
    status, self_test, error = bno.get_system_status()
    rospy.loginfo('System status:      0x{0:02X} '.format(status))

    # Print self test
    rospy.loginfo('Self test result:   0x{0:02X}'.format(self_test))
    if self_test != 0x0F:
        rospy.logwarn('WARNING: Self test result is 0x{0:02X} instead if 0x0F!'.format(self_test))

    # Print out an error if system status is in error mode.
    if status == 0x01:
        rospy.logerr('System error:    {0}'.format(error))
        rospy.loginfo('See datasheet section 4.3.59 for the meaning.')

    # Print BNO055 software revision and other diagnostic data.
    sw, bl, accel, mag, gyro = bno.get_revision()
    rospy.loginfo('Software version:   {0}'.format(sw))
    rospy.loginfo('Bootloader version: {0}'.format(bl))
    rospy.loginfo('Accelerometer ID:   0x{0:02X}'.format(accel))
    rospy.loginfo('Magnetometer ID:    0x{0:02X}'.format(mag))
    rospy.loginfo('Gyroscope ID:       0x{0:02X}\n'.format(gyro))

    rospy.loginfo('Reading BNO055 data, press Ctrl-C to quit...')
else:
    rospy.logwarn("Test mode only! BNO055 not initialised.")

while not rospy.is_shutdown():
    # check for incoming messages
    # rospy.spinOnce() >> this does not exist in rospy. Unneeded since we do not receive messages? Otherwise create a listener.

    # for header time stamps
    current_time = rospy.Time.now()

    """ IMU message header (for IMU and temperature) """
    h = rospy.Header()
    h.stamp = current_time
    h.frame_id = frame_id  # "odom"
    h.seq = seq
    # increase sequence
    seq = seq + 1
    # add header to IMU message
    imu_msg.header = h

    """ IMU readings """
    # run some parts only on the real robot
    if hostname == 'minibot':
        # Read the Euler angles for heading, roll, pitch (all in degrees).
        heading, roll, pitch = bno.read_euler()

        # Read the calibration status, 0=uncalibrated and 3=fully calibrated.
        sys, gyro, accel, mag = bno.get_calibration_status()

        # Print everything out.
        # rospy.loginfo('Heading={0:0.2F} Roll={1:0.2F} Pitch={2:0.2F}\tSys_cal={3} Gyro_cal={4} Accel_cal={5} Mag_cal={6}'.format(heading, roll, pitch, sys, gyro, accel, mag))
    else:
        # test values only!
        heading, roll, pitch  = 1.0
        sys, gyro, accel, mag = 1.0

    # publish Euler values
    pubH.publish(heading)
    pubR.publish(roll)
    pubP.publish(pitch)

    # run some parts only on the real robot
    if hostname == 'minibot':
        # Read orientation as a quaternion:
        x,y,z,w = bno.read_quaternion()
    else:
        # test values only!
        x,y,z,w = 1.0

    imu_msg.orientation.x = x # is this the pose then?
    imu_msg.orientation.y = y
    imu_msg.orientation.z = z
    imu_msg.orientation.w = w
    # Print
    # rospy.loginfo('Quaternion: x={} y={} z={} w={}'.format(imu_msg.orientation.x, imu_msg.orientation.y, imu_msg.orientation.z, imu_msg.orientation.w))

    # run some parts only on the real robot
    if hostname == 'minibot':
        # Gyroscope data (in degrees per second):
        xg,yg,zg = bno.read_gyroscope()
    else:
        # test values only!
        xg,yg,zg = 1.0

    imu_msg.angular_velocity.x = xg;
    imu_msg.angular_velocity.y = yg;
    imu_msg.angular_velocity.z = zg;
    # Print
    rospy.loginfo('Gyroscope: x={} y={} z={}'.format(imu_msg.angular_velocity.x, imu_msg.angular_velocity.y, imu_msg.angular_velocity.z))

    # run some parts only on the real robot
    if hostname == 'minibot':
        # Accelerometer data (in meters per second squared):
        xa,ya,za = bno.read_accelerometer()
    else:
        # test values only!
        xa,ya,za = 1.0

    imu_msg.linear_acceleration.x = xa;
    imu_msg.linear_acceleration.y = ya;
    imu_msg.linear_acceleration.z = za;
    # Print
    rospy.loginfo('Accelerometer: x={} y={} z={}'.format(imu_msg.linear_acceleration.x, imu_msg.linear_acceleration.y, imu_msg.linear_acceleration.z))

    #
    # publish message
    #
    pubImu.publish(imu_msg)

    """ Temperature in degrees Celsius """
    # run some parts only on the real robot
    if hostname == 'minibot':
        temp = bno.read_temp()
    else:
        # test values only!
        temp = 1.0

    # Print
    # rospy.loginfo('Temperature: {}°C'.format(temp))

    # add header to temperature message
    temp_msg.header = h

    #
    # publish message
    #
    temp_msg.temperature = temp
    pubTemp.publish(temp_msg)

    # Other values you can optionally read:
    # Magnetometer data (in micro-Teslas):
    #x,y,z = bno.read_magnetometer()
    # Linear acceleration data (i.e. acceleration from movement, not gravity--
    # returned in meters per second squared):
    #x,y,z = bno.read_linear_acceleration()
    # Gravity acceleration data (i.e. acceleration just from gravity--returned
    # in meters per second squared):
    #x,y,z = bno.read_gravity()

    """ tf stuff """
    # since all odometry is 6DOF we'll need a quaternion created from yaw
    # org: geometry_msgs::Quaternion odom_quat = tf::createQuaternionMsgFromYaw(z)
    odom_quat = tf.createQuaternionMsgFromYaw(z)

    # first, we'll publish the transform over tf
    # org: geometry_msgs::TransformStamped odom_trans
    odom_trans = geometry_msgs.msg.TransformStamped()

    odom_trans.header.stamp = current_time
    odom_trans.header.frame_id = "odom"
    odom_trans.child_frame_id = "base_link"

    odom_trans.transform.translation.x = x
    odom_trans.transform.translation.y = y
    odom_trans.transform.translation.z = 0.0
    odom_trans.transform.rotation = odom_quat

    #
    # publish/send the transform
    #
    odom_broadcaster.sendTransform(odom_trans)

    """ odom stuff """
    # next, we'll publish the odometry message over ROS
    # org: nav_msgs::Odometry odom;
    odom = nav_msgs.msg.Odometry()

    odom.header.stamp = current_time
    odom.header.frame_id = "odom"

    # set the position
    odom.pose.pose.position.x = x
    odom.pose.pose.position.y = y
    odom.pose.pose.position.z = 0.0
    odom.pose.pose.orientation = odom_quat

    # set the velocity
    odom.child_frame_id = "base_link"
    # linear acceleration from IMU (Accelerometer)
    odom.twist.twist.linear.x = xa
    odom.twist.twist.linear.y = ya
    # angular velocity from IMU (Gyroscope)
    odom.twist.twist.angular.z = zg

    #
    # publish the message
    #
    odom_pub.publish(odom);

    # Sleep for a second until the next reading.
    rospy.sleep(sleepTime)
<commit_msg>fixed test value initialisation<commit_after>#!/usr/bin/env python
# coding=utf-8

"""
ROS node for publishing "/imu/data" and "odom" (nav_msgs/Odometry) topics,
using the Adafruit BNO055 9DOF board.

Author:  Markus Knapp, 2017
Website: https://direcs.de
"""
#
# Adafruit BNO055 code:
#
# Copyright (c) 2015 Adafruit Industries
# Original-Author: Tony DiCola
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
import logging
import sys
import time

import rospy
import math

from std_msgs.msg import Header, Float32
from sensor_msgs.msg import Imu, Temperature

# see also http://wiki.ros.org/navigation/Tutorials/RobotSetup/Odom:
# "The nav_msgs/Odometry message stores an estimate of the position
#  and velocity of a robot in free space"
from nav_msgs.msg import Odometry
# for the tf broadcaster
import tf

# sleep time for this node in seconds
sleepTime = 0.25

# for getting the hostname of the underlying system
import socket
# showing hostname
hostname = socket.gethostname()
rospy.loginfo("Running on host %s.", hostname)
if hostname != 'minibot':
    rospy.logwarn("Test mode only due to other host. Skipping all I2C staff!")

# the odometry topic
pubOdom = rospy.Publisher('odom', Odometry, queue_size=50)
# the tf broadcaster
# "any odometry source must publish information about the coordinate frame that it manages"
odomBroadcaster = tf.TransformBroadcaster() # "br" in tf_broadcaster.py

# Euler topics
pubH = rospy.Publisher('heading', Float32, queue_size=1)
pubR = rospy.Publisher('roll',    Float32, queue_size=1)
pubP = rospy.Publisher('pitch',   Float32, queue_size=1)
# Temperature
pubTemp = rospy.Publisher('temperature',   Temperature, queue_size=1)
# ROS IMU format messages
pubImu  = rospy.Publisher('imu/data', Imu, queue_size=1)

# header frame for odometry message
frame_id = 'odom' #
child_id = 'base_link' # normally the coordinate frame of the mobile base, so base_link.
seq = 0

# define temperature message
temp_msg = Temperature()
# ignore the covariance data, it is optional
temp_msg.variance = 0

# define IMU message
imu_msg = Imu()
# ignore the covariance data, it is optional
imu_msg.orientation_covariance[0]         = -1
imu_msg.angular_velocity_covariance[0]    = -1
imu_msg.linear_acceleration_covariance[0] = -1

# initialise the ROS node
rospy.init_node('bno055_node')

# run some parts only on the real robot
if hostname == 'minibot':
    # load library
    from Adafruit_BNO055 import BNO055

    # Create and configure the BNO sensor connection.
    # Using I2C without a RST pin
    bno = BNO055.BNO055()

    # Initialize the BNO055 and stop if something went wrong.
    if not bno.begin():
        rospy.logerr("Failed to initialize BNO055! Is the sensor connected?")

    # Print system status
    status, self_test, error = bno.get_system_status()
    rospy.loginfo('System status:      0x{0:02X} '.format(status))

    # Print self test
    rospy.loginfo('Self test result:   0x{0:02X}'.format(self_test))
    if self_test != 0x0F:
        rospy.logwarn('WARNING: Self test result is 0x{0:02X} instead if 0x0F!'.format(self_test))

    # Print out an error if system status is in error mode.
    if status == 0x01:
        rospy.logerr('System error:    {0}'.format(error))
        rospy.loginfo('See datasheet section 4.3.59 for the meaning.')

    # Print BNO055 software revision and other diagnostic data.
    sw, bl, accel, mag, gyro = bno.get_revision()
    rospy.loginfo('Software version:   {0}'.format(sw))
    rospy.loginfo('Bootloader version: {0}'.format(bl))
    rospy.loginfo('Accelerometer ID:   0x{0:02X}'.format(accel))
    rospy.loginfo('Magnetometer ID:    0x{0:02X}'.format(mag))
    rospy.loginfo('Gyroscope ID:       0x{0:02X}\n'.format(gyro))

    rospy.loginfo('Reading BNO055 data, press Ctrl-C to quit...')
else:
    rospy.logwarn("Test mode only! BNO055 not initialised.")

while not rospy.is_shutdown():
    # check for incoming messages
    # rospy.spinOnce() >> this does not exist in rospy. Unneeded since we do not receive messages? Otherwise create a listener.

    # for header time stamps
    current_time = rospy.Time.now()

    """ IMU message header (for IMU and temperature) """
    h = rospy.Header()
    h.stamp = current_time
    h.frame_id = frame_id  # "odom"
    h.seq = seq
    # increase sequence
    seq = seq + 1
    # add header to IMU message
    imu_msg.header = h

    """ IMU readings """
    # run some parts only on the real robot
    if hostname == 'minibot':
        # Read the Euler angles for heading, roll, pitch (all in degrees).
        heading, roll, pitch = bno.read_euler()

        # Read the calibration status, 0=uncalibrated and 3=fully calibrated.
        sys, gyro, accel, mag = bno.get_calibration_status()

        # Print everything out.
        # rospy.loginfo('Heading={0:0.2F} Roll={1:0.2F} Pitch={2:0.2F}\tSys_cal={3} Gyro_cal={4} Accel_cal={5} Mag_cal={6}'.format(heading, roll, pitch, sys, gyro, accel, mag))
    else:
        # test values only!
        heading = 1.0
        roll    = 1.0
        pitch   = 1.0
        sys     = 1.0
        gyro    = 1.0
        accel   = 1.0
        mag     = 1.0

    # publish Euler values
    pubH.publish(heading)
    pubR.publish(roll)
    pubP.publish(pitch)

    # run some parts only on the real robot
    if hostname == 'minibot':
        # Read orientation as a quaternion:
        x,y,z,w = bno.read_quaternion()
    else:
        # test values only!
        x = 1.0
        y = 1.0
        z = 1.0
        w = 1.0

    imu_msg.orientation.x = x # is this the pose then?
    imu_msg.orientation.y = y
    imu_msg.orientation.z = z
    imu_msg.orientation.w = w
    # Print
    # rospy.loginfo('Quaternion: x={} y={} z={} w={}'.format(imu_msg.orientation.x, imu_msg.orientation.y, imu_msg.orientation.z, imu_msg.orientation.w))

    # run some parts only on the real robot
    if hostname == 'minibot':
        # Gyroscope data (in degrees per second):
        xg,yg,zg = bno.read_gyroscope()
    else:
        # test values only!
        xg = 1.0
        yg = 1.0
        zg = 1.0

    imu_msg.angular_velocity.x = xg;
    imu_msg.angular_velocity.y = yg;
    imu_msg.angular_velocity.z = zg;
    # Print
    rospy.loginfo('Gyroscope: x={} y={} z={}'.format(imu_msg.angular_velocity.x, imu_msg.angular_velocity.y, imu_msg.angular_velocity.z))

    # run some parts only on the real robot
    if hostname == 'minibot':
        # Accelerometer data (in meters per second squared):
        xa,ya,za = bno.read_accelerometer()
    else:
        # test values only!
        xa = 1.0
        ya = 1.0
        za = 1.0

    imu_msg.linear_acceleration.x = xa;
    imu_msg.linear_acceleration.y = ya;
    imu_msg.linear_acceleration.z = za;
    # Print
    rospy.loginfo('Accelerometer: x={} y={} z={}'.format(imu_msg.linear_acceleration.x, imu_msg.linear_acceleration.y, imu_msg.linear_acceleration.z))

    #
    # publish message
    #
    pubImu.publish(imu_msg)

    """ Temperature in degrees Celsius """
    # run some parts only on the real robot
    if hostname == 'minibot':
        temp = bno.read_temp()
    else:
        # test values only!
        temp = 1.0

    # Print
    # rospy.loginfo('Temperature: {}°C'.format(temp))

    # add header to temperature message
    temp_msg.header = h

    #
    # publish message
    #
    temp_msg.temperature = temp
    pubTemp.publish(temp_msg)

    # Other values you can optionally read:
    # Magnetometer data (in micro-Teslas):
    #x,y,z = bno.read_magnetometer()
    # Linear acceleration data (i.e. acceleration from movement, not gravity--
    # returned in meters per second squared):
    #x,y,z = bno.read_linear_acceleration()
    # Gravity acceleration data (i.e. acceleration just from gravity--returned
    # in meters per second squared):
    #x,y,z = bno.read_gravity()

    """ tf stuff """
    # since all odometry is 6DOF we'll need a quaternion created from yaw
    # org: geometry_msgs::Quaternion odom_quat = tf::createQuaternionMsgFromYaw(z)
    odom_quat = tf.createQuaternionMsgFromYaw(z)

    # first, we'll publish the transform over tf
    # org: geometry_msgs::TransformStamped odom_trans
    odom_trans = geometry_msgs.msg.TransformStamped()

    odom_trans.header.stamp = current_time
    odom_trans.header.frame_id = "odom"
    odom_trans.child_frame_id = "base_link"

    odom_trans.transform.translation.x = x
    odom_trans.transform.translation.y = y
    odom_trans.transform.translation.z = 0.0
    odom_trans.transform.rotation = odom_quat

    #
    # publish/send the transform
    #
    odom_broadcaster.sendTransform(odom_trans)

    """ odom stuff """
    # next, we'll publish the odometry message over ROS
    # org: nav_msgs::Odometry odom;
    odom = nav_msgs.msg.Odometry()

    odom.header.stamp = current_time
    odom.header.frame_id = "odom"

    # set the position
    odom.pose.pose.position.x = x
    odom.pose.pose.position.y = y
    odom.pose.pose.position.z = 0.0
    odom.pose.pose.orientation = odom_quat

    # set the velocity
    odom.child_frame_id = "base_link"
    # linear acceleration from IMU (Accelerometer)
    odom.twist.twist.linear.x = xa
    odom.twist.twist.linear.y = ya
    # angular velocity from IMU (Gyroscope)
    odom.twist.twist.angular.z = zg

    #
    # publish the message
    #
    odom_pub.publish(odom);

    # Sleep for a second until the next reading.
    rospy.sleep(sleepTime)
<|endoftext|>


<commit_before># Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Pull YouTube data for the placements in the Google Ads report."""
import base64
from datetime import datetime
import json
import logging
import math
import os
import sys
from typing import Any, Dict, List, Tuple
import google.auth
import google.auth.credentials
from google.cloud import bigquery
from google.cloud import translate_v2 as translate
from googleapiclient.discovery import build
import jsonschema
import pandas as pd
import numpy as np

logging.basicConfig(stream=sys.stdout)
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

# The Google Cloud project containing the BigQuery dataset
GOOGLE_CLOUD_PROJECT = os.environ.get('GOOGLE_CLOUD_PROJECT')

# The schema of the JSON in the event payload
message_schema = {
    'type': 'object',
    'properties': {
        'customer_id': {'type': 'string'},
    },
    'required': ['customer_id', ]
}

def main(event: Dict[str, Any], context: Dict[str, Any]) -> None:
    """The entry point: extract the data from the payload and starts the job.

    The pub/sub message must match the message_schema object above.

    Args:
        event: A dictionary representing the event data payload.
        context: An object containing metadata about the event.

    Raises:
        jsonschema.exceptions.ValidationError if the message from pub/sub is not
        what is expected.
    """
    del context
    logger.info('Google Ads Exclusion service triggered.')
    logger.info('Message: %s', event)
    message = base64.b64decode(event['data']).decode('utf-8')
    logger.info('Decoded message: %s', message)
    message_json = json.loads(message)
    logger.info('JSON message: %s', message_json)

    # Will raise jsonschema.exceptions.ValidationError if the schema is invalid
    jsonschema.validate(instance=message_json, schema=message_schema)

    run(message_json.get('customer_id'))

    logger.info('Done')

def run(customer_id: str) -> None:
    """Orchestration to pull YouTube data and output it to BigQuery.

    Args:
        customer_id: the Google Ads customer ID to process.
    """
    credentials = get_auth_credentials()
    channel_ids = get_placements_query(customer_id, credentials)
    get_youtube_dataframe(channel_ids, credentials)
    logger.info('Done')

def get_auth_credentials() -> google.auth.credentials.Credentials:
    """Return credentials for Google APIs."""
    credentials, project_id = google.auth.default()
    return credentials

def get_placements_query(
        customer_id: str,
        credentials: google.auth.credentials.Credentials
) -> List[str]:
    """Get the placements from the Google Ads report in BigQuery.

    Args:
        customer_id: the Google Ads customer ID to process.
        credentials: Google Auth credentials

    Returns:
        A list of placement IDs that need to be pulled from YouTube
    """
    logger.info('Getting Placements from Google Ads')
    logger.info('Connecting to: %s BigQuery', GOOGLE_CLOUD_PROJECT)
    client = bigquery.Client(
        project=GOOGLE_CLOUD_PROJECT, credentials=credentials)

    query = f"""
      WITH YouTube AS (
        SELECT
          *
        FROM
          `ads_placement_excluder.*`
        WHERE
          _TABLE_SUFFIX = 'youtube_channels'
      )
      SELECT DISTINCT
        Ads.placement
      FROM
        `ads_placement_excluder.google_ads_placement_report_{customer_id}` AS Ads
      LEFT JOIN
        YouTube USING(placement)
      WHERE
        YouTube.placement IS NULL
    """
    logger.info('Running query: %s', query)
    rows = client.query(query).result()
    channel_ids = []
    for row in rows:
        channel_ids.append(row.placement)
    logger.info('Received %s channel_ids', len(channel_ids))
    return channel_ids

def get_youtube_dataframe(
        channel_ids: List[str],
        credentials: google.auth.credentials.Credentials
) -> None:
    """Pull information on each of the channels provide from the YouTube API.

    The YouTube API only allows pulling up to 50 channels in each request, so
    multiple requests have to be made to pull all the data. See the docs for
    more details:
    https://developers.google.com/youtube/v3/docs/channels/list

    Args:
        channel_ids: the channel IDs to pull the info on from YouTube
        credentials: Google Auth credentials
    """
    logger.info('Getting YouTube data for channel IDs')
    # Maximum number of channels per YouTube request. See:
    # https://developers.google.com/youtube/v3/docs/channels/list
    chunk_size = 50
    chunks = split_list_to_chunks(channel_ids, chunk_size)
    number_of_chunks = len(chunks)

    logger.info('Connecting to the youtube API')
    youtube = build('youtube', 'v3', credentials=credentials)

    for i, chunk in enumerate(chunks):
        logger.info(f'Processing chunk {i + 1} of {number_of_chunks}')
        chunk_list = list(chunk)
        request = youtube.channels().list(
            part='id, statistics, snippet, brandingSettings',
            id=chunk_list,
            maxResults=chunk_size)
        response = request.execute()
        channels = process_youtube_response(response, chunk_list)
        youtube_df = pd.DataFrame(channels, columns=[
            'placement',
            'viewCount',
            'videoCount',
            'subscriberCount',
            'title',
            'title_language',
            'title_language_confidence',
            'country',
            'defaultLanguage',
            'defaultLanguageBrand'
        ])
        youtube_df['datetime_updated'] = datetime.now()
        youtube_df = youtube_df.astype({
            'viewCount': pd.Int64Dtype(),
            'videoCount': pd.Int64Dtype(),
            'subscriberCount': pd.Int64Dtype(),
            'title_language_confidence': 'float',
        })
        write_results_to_bigquery(youtube_df)
    logger.info('YouTube channel info complete')

def split_list_to_chunks(
        lst: List[Any], max_size_of_chunk: int) -> List[np.ndarray]:
    """Split the list into X chunks with the maximum size as specified.

    Args:
        lst: The list to be split into chunks
        max_size_of_chunk: the maximum number of elements that should be in a
            chunk.

    Returns:
        A list containing numpy array chunks of the original list.
    """
    logger.info('Splitting list into chunks')
    num_of_chunks = math.ceil(len(lst) / max_size_of_chunk)
    chunks = np.array_split(lst, num_of_chunks)
    logger.info('Split list into %i chunks', num_of_chunks)
    return chunks

def process_youtube_response(
        response: Dict[str, Any],
        channel_ids: List[str],
) -> List[List[Any]]:
    """Process the YouTube response to extract the required information.

    Args:
        response: The YouTube channels list response
            https://developers.google.com/youtube/v3/docs/channels/list#response
        channel_ids: A list of the channel IDs passed in the request

    Returns:
        A list of dicts where each dict represents data from one channel
    """
    logger.info('Processing youtube response')
    data = []
    if response.get('pageInfo').get('totalResults') == 0:
        logger.warning('The YouTube response has no results: %s', response)
        logger.warning(channel_ids)
        return data
    for channel in response['items']:
        title = channel.get('snippet').get('title', '')
        title_language, confidence = detect_language(title)
        data.append([
            channel.get('id'),
            channel.get('statistics').get('viewCount', None),
            channel.get('statistics').get('subscriberCount', None),
            channel.get('statistics').get('videoCount', None),
            title,
            title_language,
            confidence,
            channel.get('snippet').get('country', ''),
            channel.get('snippet').get('defaultLanguage', ''),
            channel.get('brandingSettings').get('defaultLanguage', ''),
        ])
    return data

def detect_language(text: str) -> Tuple[str, float]:
    """Detects the text's language.

    Args:
        text: the text to base the translation off of

    Returns:
        A tuple containing the language and the confidence.
    """
    logger.debug('Detecting language for %s', text)
    translate_client = translate.Client()
    result = translate_client.detect_language(text)
    return result['language'], result['confidence']

def write_results_to_bigquery(youtube_df: pd.DataFrame) -> None:
    """Write the YouTube dataframe to BigQuery

    Args:
        youtube_df: the dataframe based on the YouTube data.
    """
    logger.info('Writing the results to BigQuery in: %s', GOOGLE_CLOUD_PROJECT)
    logger.info('There are %s rows', len(youtube_df.index))
    destination_table = f'ads_placement_excluder.youtube_channels'
    logger.info('Destination is: %s', destination_table)
    youtube_df.to_gbq(
        destination_table=destination_table,
        project_id=GOOGLE_CLOUD_PROJECT,
        if_exists='append',
    )
<commit_msg>Configure YouTube service to handle no channel IDs<commit_after># Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Pull YouTube data for the placements in the Google Ads report."""
import base64
from datetime import datetime
import json
import logging
import math
import os
import sys
from typing import Any, Dict, List, Tuple
import google.auth
import google.auth.credentials
from google.cloud import bigquery
from google.cloud import translate_v2 as translate
from googleapiclient.discovery import build
import jsonschema
import pandas as pd
import numpy as np

logging.basicConfig(stream=sys.stdout)
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

# The Google Cloud project containing the BigQuery dataset
GOOGLE_CLOUD_PROJECT = os.environ.get('GOOGLE_CLOUD_PROJECT')

# The schema of the JSON in the event payload
message_schema = {
    'type': 'object',
    'properties': {
        'customer_id': {'type': 'string'},
    },
    'required': ['customer_id', ]
}

def main(event: Dict[str, Any], context: Dict[str, Any]) -> None:
    """The entry point: extract the data from the payload and starts the job.

    The pub/sub message must match the message_schema object above.

    Args:
        event: A dictionary representing the event data payload.
        context: An object containing metadata about the event.

    Raises:
        jsonschema.exceptions.ValidationError if the message from pub/sub is not
        what is expected.
    """
    del context
    logger.info('YouTube channel service triggered.')
    logger.info('Message: %s', event)
    message = base64.b64decode(event['data']).decode('utf-8')
    logger.info('Decoded message: %s', message)
    message_json = json.loads(message)
    logger.info('JSON message: %s', message_json)

    # Will raise jsonschema.exceptions.ValidationError if the schema is invalid
    jsonschema.validate(instance=message_json, schema=message_schema)

    run(message_json.get('customer_id'))

    logger.info('Done')

def run(customer_id: str) -> None:
    """Orchestration to pull YouTube data and output it to BigQuery.

    Args:
        customer_id: the Google Ads customer ID to process.
    """
    credentials = get_auth_credentials()
    channel_ids = get_placements_query(customer_id, credentials)
    if len(channel_ids) > 0:
        get_youtube_dataframe(channel_ids, credentials)
    else:
        logger.info('No channel IDs to process')
    logger.info('Done')

def get_auth_credentials() -> google.auth.credentials.Credentials:
    """Return credentials for Google APIs."""
    credentials, project_id = google.auth.default()
    return credentials

def get_placements_query(
        customer_id: str,
        credentials: google.auth.credentials.Credentials
) -> List[str]:
    """Get the placements from the Google Ads report in BigQuery.

    Args:
        customer_id: the Google Ads customer ID to process.
        credentials: Google Auth credentials

    Returns:
        A list of placement IDs that need to be pulled from YouTube
    """
    logger.info('Getting Placements from Google Ads')
    logger.info('Connecting to: %s BigQuery', GOOGLE_CLOUD_PROJECT)
    client = bigquery.Client(
        project=GOOGLE_CLOUD_PROJECT, credentials=credentials)

    query = f"""
      WITH YouTube AS (
        SELECT
          *
        FROM
          `ads_placement_excluder.*`
        WHERE
          _TABLE_SUFFIX = 'youtube_channels'
      )
      SELECT DISTINCT
        Ads.placement
      FROM
        `ads_placement_excluder.google_ads_placement_report_{customer_id}` AS Ads
      LEFT JOIN
        YouTube USING(placement)
      WHERE
        YouTube.placement IS NULL
    """
    logger.info('Running query: %s', query)
    rows = client.query(query).result()
    channel_ids = []
    for row in rows:
        channel_ids.append(row.placement)
    logger.info('Received %s channel_ids', len(channel_ids))
    return channel_ids

def get_youtube_dataframe(
        channel_ids: List[str],
        credentials: google.auth.credentials.Credentials
) -> None:
    """Pull information on each of the channels provide from the YouTube API.

    The YouTube API only allows pulling up to 50 channels in each request, so
    multiple requests have to be made to pull all the data. See the docs for
    more details:
    https://developers.google.com/youtube/v3/docs/channels/list

    Args:
        channel_ids: the channel IDs to pull the info on from YouTube
        credentials: Google Auth credentials
    """
    logger.info('Getting YouTube data for channel IDs')
    # Maximum number of channels per YouTube request. See:
    # https://developers.google.com/youtube/v3/docs/channels/list
    chunk_size = 50
    chunks = split_list_to_chunks(channel_ids, chunk_size)
    number_of_chunks = len(chunks)

    logger.info('Connecting to the youtube API')
    youtube = build('youtube', 'v3', credentials=credentials)

    for i, chunk in enumerate(chunks):
        logger.info(f'Processing chunk {i + 1} of {number_of_chunks}')
        chunk_list = list(chunk)
        request = youtube.channels().list(
            part='id, statistics, snippet, brandingSettings',
            id=chunk_list,
            maxResults=chunk_size)
        response = request.execute()
        channels = process_youtube_response(response, chunk_list)
        youtube_df = pd.DataFrame(channels, columns=[
            'placement',
            'viewCount',
            'videoCount',
            'subscriberCount',
            'title',
            'title_language',
            'title_language_confidence',
            'country',
            'defaultLanguage',
            'defaultLanguageBrand'
        ])
        youtube_df['datetime_updated'] = datetime.now()
        youtube_df = youtube_df.astype({
            'viewCount': pd.Int64Dtype(),
            'videoCount': pd.Int64Dtype(),
            'subscriberCount': pd.Int64Dtype(),
            'title_language_confidence': 'float',
        })
        write_results_to_bigquery(youtube_df)
    logger.info('YouTube channel info complete')

def split_list_to_chunks(
        lst: List[Any], max_size_of_chunk: int) -> List[np.ndarray]:
    """Split the list into X chunks with the maximum size as specified.

    Args:
        lst: The list to be split into chunks
        max_size_of_chunk: the maximum number of elements that should be in a
            chunk.

    Returns:
        A list containing numpy array chunks of the original list.
    """
    logger.info('Splitting list into chunks')
    num_of_chunks = math.ceil(len(lst) / max_size_of_chunk)
    chunks = np.array_split(lst, num_of_chunks)
    logger.info('Split list into %i chunks', num_of_chunks)
    return chunks

def process_youtube_response(
        response: Dict[str, Any],
        channel_ids: List[str],
) -> List[List[Any]]:
    """Process the YouTube response to extract the required information.

    Args:
        response: The YouTube channels list response
            https://developers.google.com/youtube/v3/docs/channels/list#response
        channel_ids: A list of the channel IDs passed in the request

    Returns:
        A list of dicts where each dict represents data from one channel
    """
    logger.info('Processing youtube response')
    data = []
    if response.get('pageInfo').get('totalResults') == 0:
        logger.warning('The YouTube response has no results: %s', response)
        logger.warning(channel_ids)
        return data
    for channel in response['items']:
        title = channel.get('snippet').get('title', '')
        title_language, confidence = detect_language(title)
        data.append([
            channel.get('id'),
            channel.get('statistics').get('viewCount', None),
            channel.get('statistics').get('subscriberCount', None),
            channel.get('statistics').get('videoCount', None),
            title,
            title_language,
            confidence,
            channel.get('snippet').get('country', ''),
            channel.get('snippet').get('defaultLanguage', ''),
            channel.get('brandingSettings').get('defaultLanguage', ''),
        ])
    return data

def detect_language(text: str) -> Tuple[str, float]:
    """Detects the text's language.

    Args:
        text: the text to base the translation off of

    Returns:
        A tuple containing the language and the confidence.
    """
    logger.debug('Detecting language for %s', text)
    translate_client = translate.Client()
    result = translate_client.detect_language(text)
    return result['language'], result['confidence']

def write_results_to_bigquery(youtube_df: pd.DataFrame) -> None:
    """Write the YouTube dataframe to BigQuery

    Args:
        youtube_df: the dataframe based on the YouTube data.
    """
    logger.info('Writing the results to BigQuery in: %s', GOOGLE_CLOUD_PROJECT)
    logger.info('There are %s rows', len(youtube_df.index))
    destination_table = f'ads_placement_excluder.youtube_channels'
    logger.info('Destination is: %s', destination_table)
    youtube_df.to_gbq(
        destination_table=destination_table,
        project_id=GOOGLE_CLOUD_PROJECT,
        if_exists='append',
    )
<|endoftext|>


<commit_before>from zeit.edit.rule import glob
import itertools
import zeit.content.cp.interfaces
import zope.component
import zope.interface

class CenterPageValidator(zeit.edit.rule.RecursiveValidator):

    zope.component.adapts(zeit.content.cp.interfaces.ICenterPage)

    @property
    def children(self):
        areas = self.context.values()
        return itertools.chain(areas, *[a.values() for a in areas])

@glob(zeit.content.cp.interfaces.IElement)
def region(context):
    return zeit.content.cp.interfaces.IRegion(context).__name__

@glob(zeit.content.cp.interfaces.ITeaserBlock)
def layout(context):
    return context.layout.id

@glob(zope.interface.Interface)
def layout(context):
    return None

@glob(zeit.content.cp.interfaces.IBlock)
def content(context):
    return list(
        zeit.content.cp.interfaces.ICMSContentIterable(context, []))

@glob(zope.interface.Interface)
def cp_type(context):
    cp = zeit.content.cp.interfaces.ICenterPage(context, None)
    if cp is None:
        return None
    return cp.type
<commit_msg>Fix rule glob definition for ``region``, it needs the typical "None if not applicable" variant<commit_after>from zeit.edit.rule import glob
import itertools
import zeit.content.cp.interfaces
import zope.component
import zope.interface

class CenterPageValidator(zeit.edit.rule.RecursiveValidator):

    zope.component.adapts(zeit.content.cp.interfaces.ICenterPage)

    @property
    def children(self):
        areas = self.context.values()
        return itertools.chain(areas, *[a.values() for a in areas])

@glob(zeit.content.cp.interfaces.IElement)
def region(context):
    return zeit.content.cp.interfaces.IRegion(context).__name__

@glob(zope.interface.Interface)
def region(context):
    return None

@glob(zeit.content.cp.interfaces.ITeaserBlock)
def layout(context):
    return context.layout.id

@glob(zope.interface.Interface)
def layout(context):
    return None

@glob(zeit.content.cp.interfaces.IBlock)
def content(context):
    return list(
        zeit.content.cp.interfaces.ICMSContentIterable(context, []))

@glob(zope.interface.Interface)
def cp_type(context):
    cp = zeit.content.cp.interfaces.ICenterPage(context, None)
    if cp is None:
        return None
    return cp.type
<|endoftext|>


<commit_before>from zeit.cms.content.interfaces import WRITEABLE_ALWAYS
from zeit.retresco.tag import Tag
import grokcore.component as grok
import logging
import lxml.etree
import lxml.objectify
import zeit.cms.content.dav
import zeit.cms.tagging.interfaces
import zeit.connector.interfaces
import zeit.retresco.interfaces
import zope.component

NAMESPACE = "http://namespaces.zeit.de/CMS/tagging"
KEYWORD_PROPERTY = ('rankedTags', NAMESPACE)
DISABLED_PROPERTY = ('disabled', NAMESPACE)
DISABLED_SEPARATOR = '\x09'
PINNED_PROPERTY = ('pinned', NAMESPACE)

# To speed up indexing we do not checkout resources to update properties.
# This is why we make the keyword property writeable.
property_manager = zeit.cms.content.liveproperty.LiveProperties
property_manager.register_live_property(
    KEYWORD_PROPERTY[0], NAMESPACE, WRITEABLE_ALWAYS)
property_manager.register_live_property(
    DISABLED_PROPERTY[0], NAMESPACE, WRITEABLE_ALWAYS)

log = logging.getLogger(__name__)

class Tagger(zeit.cms.content.dav.DAVPropertiesAdapter):
    """Serializes keywords to an XML structure and stores it in a DAV-Property.

    The serialization format (including pinned and disabled keywords) is
    actually backwards compatible with zeit.intrafind, but since the TMS uses
    different IDs for keywords, switching to retresco means that
    pinned+disabled keywords will effectively be reset. (Keywords that are
    assigned to content already will remain usable, though, since the label is
    pretty much the only interesting property.)

    We need to read/write all tags from/to xml for every operation, since
    adapting to ``ITagger`` returns a *different* Tagger each time, so we
    cannot (easily) cache anything on the Tagger instance.

    There are also '<rankedTags>' in the XML of ``context`` but these are
    written by ``zeit.cms.tagging.tag.add_ranked_tags_to_head``. At this point,
    the information about pinned and disabled tags is omitted, as it is only
    needed in vivi.

    """

    grok.implements(zeit.cms.tagging.interfaces.ITagger)

    def __iter__(self):
        tags = self.to_xml()
        if tags is None:
            return iter(())
        return (Tag(x.text, x.get('type', '')).code
                for x in tags.iterchildren())

    def __len__(self):
        return len(list(self.__iter__()))

    def __getitem__(self, key):
        node = self._find_tag_node(key)
        tag = self._create_tag(unicode(node), node.get('type', ''))
        return tag

    def __setitem__(self, key, value):
        tags = self.to_xml()
        if tags is None:
            E = lxml.objectify.ElementMaker(namespace=NAMESPACE)
            root = E.rankedTags()
            tags = E.rankedTags()
            root.append(tags)
        # XXX the handling of namespaces here seems chaotic
        E = lxml.objectify.ElementMaker()
        tags.append(E.tag(value.label, type=value.entity_type or ''))
        dav = zeit.connector.interfaces.IWebDAVProperties(self)
        dav[KEYWORD_PROPERTY] = lxml.etree.tostring(tags.getroottree())

    def values(self):
        tags = self.to_xml()
        if tags is None:
            return iter(())
        return (self._create_tag(unicode(node), node.get('type', ''))
                for node in tags.iterchildren())

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def keys(self):
        raise NotImplementedError()

    def items(self):
        raise NotImplementedError()

    def __contains__(self, key):
        return key in list(self)

    def updateOrder(self, keys):
        keys = list(keys)  # in case we've got a generator
        if set(keys) != set(self):
            raise ValueError(
                'Must pass in the same keys already present %r, not %r'
                % (list(self), keys))
        orderd = []
        tags = self.to_xml()
        for key in keys:
            tag = self._find_tag_node(key, tags)
            orderd.append(tag)
            tags.remove(tag)
        tags.extend(orderd)
        dav = zeit.connector.interfaces.IWebDAVProperties(self)
        dav[KEYWORD_PROPERTY] = lxml.etree.tostring(tags.getroottree())

    def __delitem__(self, key):
        node = self._find_tag_node(key)
        node.getparent().remove(node)
        dav = zeit.connector.interfaces.IWebDAVProperties(self)
        dav[KEYWORD_PROPERTY] = lxml.etree.tostring(node.getroottree())
        self._disable(key)

    def set_pinned(self, keys):
        dav = zeit.connector.interfaces.IWebDAVProperties(self)
        dav[PINNED_PROPERTY] = DISABLED_SEPARATOR.join(keys)

    @property
    def pinned(self):
        dav = zeit.connector.interfaces.IWebDAVProperties(self)
        value = dav.get(PINNED_PROPERTY, '')
        if not value:
            return ()
        return tuple(value.split(DISABLED_SEPARATOR))

    @property
    def disabled(self):
        dav = zeit.connector.interfaces.IWebDAVProperties(self)
        disabled = dav.get(DISABLED_PROPERTY)
        if not disabled:
            return ()
        return tuple(disabled.split(DISABLED_SEPARATOR))

    def _disable(self, key):
        dav = zeit.connector.interfaces.IWebDAVProperties(self)
        disabled = self.disabled + (key,)
        dav[DISABLED_PROPERTY] = DISABLED_SEPARATOR.join(disabled)

    def to_xml(self):
        dav = zeit.connector.interfaces.IWebDAVProperties(self)
        try:
            tags = lxml.objectify.fromstring(dav.get(KEYWORD_PROPERTY, ''))
        except lxml.etree.XMLSyntaxError:
            return None
        if tags.tag != '{{{1}}}{0}'.format(*KEYWORD_PROPERTY):
            return None
        return tags.getchildren()[0]

    def _find_tag_node(self, key, tags=None):
        if tags is None:
            tags = self.to_xml()
        if tags is None:
            raise KeyError(key)

        node = [x for x in tags.iterchildren()
                if Tag(x.text, x.get('type', '')).code == key]
        if not node:
            raise KeyError(key)
        return node[0]

    def _create_tag(self, label, entity_type):
        tag = Tag(label, entity_type)
        if tag.code in self.pinned:
            tag.pinned = True
        # XXX it is not understood by the writer, whether we really need this.
        # It was kept from the former implementation in `zeit.cms.tagging`.
        tag.__parent__ = self
        return tag

    def update(self, update_with=None):
        """Update the keywords with generated keywords from retresco.

        A number of reasonable keywords are retrieved from retresco. This set
        is reduced by the disabled keywords and enriched by the pinned
        keywords. The resulting set is finally written to the DAV property.

        """
        log.info('Updating tags for %s', self.context.uniqueId)
        tms = zope.component.getUtility(zeit.retresco.interfaces.ITMS)

        if update_with is not None:
            keywords = tms.generate_keyword_list(update_with)
        else:
            keywords = tms.extract_keywords(self.context)

        E = lxml.objectify.ElementMaker(namespace=NAMESPACE)
        root = E.rankedTags()
        new_tags = E.rankedTags()
        root.append(new_tags)

        # XXX the handling of namespaces here seems chaotic
        E = lxml.objectify.ElementMaker()
        new_codes = set()
        for tag in keywords:
            if tag.code in self.disabled:
                continue
            new_codes.add(tag.code)
            new_tags.append(E.tag(tag.label, type=tag.entity_type or ''))

        old_tags = self.to_xml()
        for code in self.pinned:
            if code not in new_codes:
                try:
                    pinned_tag = self._find_tag_node(code, old_tags)
                    new_tags.append(pinned_tag)
                except KeyError:
                    pass

        dav = zeit.connector.interfaces.IWebDAVProperties(self)
        dav[KEYWORD_PROPERTY] = lxml.etree.tostring(root.getroottree())
        dav[DISABLED_PROPERTY] = u''
<commit_msg>Add writeable always properties for tags<commit_after>from zeit.cms.content.interfaces import WRITEABLE_ALWAYS
from zeit.retresco.tag import Tag
import grokcore.component as grok
import logging
import lxml.etree
import lxml.objectify
import zeit.cms.content.dav
import zeit.cms.tagging.interfaces
import zeit.connector.interfaces
import zeit.retresco.interfaces
import zope.component

NAMESPACE = "http://namespaces.zeit.de/CMS/tagging"
KEYWORD_PROPERTY = ('rankedTags', NAMESPACE)
DISABLED_PROPERTY = ('disabled', NAMESPACE)
DISABLED_SEPARATOR = '\x09'
PINNED_PROPERTY = ('pinned', NAMESPACE)

# To speed up indexing we do not checkout resources to update properties.
# This is why we make the keyword property writeable.
property_manager = zeit.cms.content.liveproperty.LiveProperties
property_manager.register_live_property(
    KEYWORD_PROPERTY[0], NAMESPACE, WRITEABLE_ALWAYS)
property_manager.register_live_property(
    DISABLED_PROPERTY[0], NAMESPACE, WRITEABLE_ALWAYS)
property_manager.register_live_property(
    KEYWORD_PROPERTY[0], NAMESPACE, WRITEABLE_ALWAYS)
property_manager.register_live_property(
    PINNED_PROPERTY[0], NAMESPACE, WRITEABLE_ALWAYS)

log = logging.getLogger(__name__)

class Tagger(zeit.cms.content.dav.DAVPropertiesAdapter):
    """Serializes keywords to an XML structure and stores it in a DAV-Property.

    The serialization format (including pinned and disabled keywords) is
    actually backwards compatible with zeit.intrafind, but since the TMS uses
    different IDs for keywords, switching to retresco means that
    pinned+disabled keywords will effectively be reset. (Keywords that are
    assigned to content already will remain usable, though, since the label is
    pretty much the only interesting property.)

    We need to read/write all tags from/to xml for every operation, since
    adapting to ``ITagger`` returns a *different* Tagger each time, so we
    cannot (easily) cache anything on the Tagger instance.

    There are also '<rankedTags>' in the XML of ``context`` but these are
    written by ``zeit.cms.tagging.tag.add_ranked_tags_to_head``. At this point,
    the information about pinned and disabled tags is omitted, as it is only
    needed in vivi.

    """

    grok.implements(zeit.cms.tagging.interfaces.ITagger)

    def __iter__(self):
        tags = self.to_xml()
        if tags is None:
            return iter(())
        return (Tag(x.text, x.get('type', '')).code
                for x in tags.iterchildren())

    def __len__(self):
        return len(list(self.__iter__()))

    def __getitem__(self, key):
        node = self._find_tag_node(key)
        tag = self._create_tag(unicode(node), node.get('type', ''))
        return tag

    def __setitem__(self, key, value):
        tags = self.to_xml()
        if tags is None:
            E = lxml.objectify.ElementMaker(namespace=NAMESPACE)
            root = E.rankedTags()
            tags = E.rankedTags()
            root.append(tags)
        # XXX the handling of namespaces here seems chaotic
        E = lxml.objectify.ElementMaker()
        tags.append(E.tag(value.label, type=value.entity_type or ''))
        dav = zeit.connector.interfaces.IWebDAVProperties(self)
        dav[KEYWORD_PROPERTY] = lxml.etree.tostring(tags.getroottree())

    def values(self):
        tags = self.to_xml()
        if tags is None:
            return iter(())
        return (self._create_tag(unicode(node), node.get('type', ''))
                for node in tags.iterchildren())

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def keys(self):
        raise NotImplementedError()

    def items(self):
        raise NotImplementedError()

    def __contains__(self, key):
        return key in list(self)

    def updateOrder(self, keys):
        keys = list(keys)  # in case we've got a generator
        if set(keys) != set(self):
            raise ValueError(
                'Must pass in the same keys already present %r, not %r'
                % (list(self), keys))
        orderd = []
        tags = self.to_xml()
        for key in keys:
            tag = self._find_tag_node(key, tags)
            orderd.append(tag)
            tags.remove(tag)
        tags.extend(orderd)
        dav = zeit.connector.interfaces.IWebDAVProperties(self)
        dav[KEYWORD_PROPERTY] = lxml.etree.tostring(tags.getroottree())

    def __delitem__(self, key):
        node = self._find_tag_node(key)
        node.getparent().remove(node)
        dav = zeit.connector.interfaces.IWebDAVProperties(self)
        dav[KEYWORD_PROPERTY] = lxml.etree.tostring(node.getroottree())
        self._disable(key)

    def set_pinned(self, keys):
        dav = zeit.connector.interfaces.IWebDAVProperties(self)
        dav[PINNED_PROPERTY] = DISABLED_SEPARATOR.join(keys)

    @property
    def pinned(self):
        dav = zeit.connector.interfaces.IWebDAVProperties(self)
        value = dav.get(PINNED_PROPERTY, '')
        if not value:
            return ()
        return tuple(value.split(DISABLED_SEPARATOR))

    @property
    def disabled(self):
        dav = zeit.connector.interfaces.IWebDAVProperties(self)
        disabled = dav.get(DISABLED_PROPERTY)
        if not disabled:
            return ()
        return tuple(disabled.split(DISABLED_SEPARATOR))

    def _disable(self, key):
        dav = zeit.connector.interfaces.IWebDAVProperties(self)
        disabled = self.disabled + (key,)
        dav[DISABLED_PROPERTY] = DISABLED_SEPARATOR.join(disabled)

    def to_xml(self):
        dav = zeit.connector.interfaces.IWebDAVProperties(self)
        try:
            tags = lxml.objectify.fromstring(dav.get(KEYWORD_PROPERTY, ''))
        except lxml.etree.XMLSyntaxError:
            return None
        if tags.tag != '{{{1}}}{0}'.format(*KEYWORD_PROPERTY):
            return None
        return tags.getchildren()[0]

    def _find_tag_node(self, key, tags=None):
        if tags is None:
            tags = self.to_xml()
        if tags is None:
            raise KeyError(key)

        node = [x for x in tags.iterchildren()
                if Tag(x.text, x.get('type', '')).code == key]
        if not node:
            raise KeyError(key)
        return node[0]

    def _create_tag(self, label, entity_type):
        tag = Tag(label, entity_type)
        if tag.code in self.pinned:
            tag.pinned = True
        # XXX it is not understood by the writer, whether we really need this.
        # It was kept from the former implementation in `zeit.cms.tagging`.
        tag.__parent__ = self
        return tag

    def update(self, update_with=None):
        """Update the keywords with generated keywords from retresco.

        A number of reasonable keywords are retrieved from retresco. This set
        is reduced by the disabled keywords and enriched by the pinned
        keywords. The resulting set is finally written to the DAV property.

        """
        log.info('Updating tags for %s', self.context.uniqueId)
        tms = zope.component.getUtility(zeit.retresco.interfaces.ITMS)

        if update_with is not None:
            keywords = tms.generate_keyword_list(update_with)
        else:
            keywords = tms.extract_keywords(self.context)

        E = lxml.objectify.ElementMaker(namespace=NAMESPACE)
        root = E.rankedTags()
        new_tags = E.rankedTags()
        root.append(new_tags)

        # XXX the handling of namespaces here seems chaotic
        E = lxml.objectify.ElementMaker()
        new_codes = set()
        for tag in keywords:
            if tag.code in self.disabled:
                continue
            new_codes.add(tag.code)
            new_tags.append(E.tag(tag.label, type=tag.entity_type or ''))

        old_tags = self.to_xml()
        for code in self.pinned:
            if code not in new_codes:
                try:
                    pinned_tag = self._find_tag_node(code, old_tags)
                    new_tags.append(pinned_tag)
                except KeyError:
                    pass

        dav = zeit.connector.interfaces.IWebDAVProperties(self)
        dav[KEYWORD_PROPERTY] = lxml.etree.tostring(root.getroottree())
        dav[DISABLED_PROPERTY] = u''
<|endoftext|>


<commit_before>##############################################################################
# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *

class PyDecorator(PythonPackage):
    """The aim of the decorator module it to simplify the usage of decorators
       for the average programmer, and to popularize decorators by showing
       various non-trivial examples."""
    homepage = "https://github.com/micheles/decorator"
    url      = "https://pypi.io/packages/source/d/decorator/decorator-4.0.9.tar.gz"

    version('4.0.9', 'f12c5651ccd707e12a0abaa4f76cd69a')

    depends_on('py-setuptools', type='build')
<commit_msg>py-decorator: new version (#9240)<commit_after>##############################################################################
# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *

class PyDecorator(PythonPackage):
    """The aim of the decorator module it to simplify the usage of decorators
       for the average programmer, and to popularize decorators by showing
       various non-trivial examples."""
    homepage = "https://github.com/micheles/decorator"
    url      = "https://pypi.io/packages/source/d/decorator/decorator-4.0.9.tar.gz"

    version('4.3.0', sha256='c39efa13fbdeb4506c476c9b3babf6a718da943dab7811c206005a4a956c080c')
    version('4.0.9', 'f12c5651ccd707e12a0abaa4f76cd69a')

    depends_on('py-setuptools', type='build')
<|endoftext|>


<commit_before>#
# Jasy - Web Tooling Framework
# Copyright 2013 Sebastian Werner
#

import json, re, sys
import jasy.style.Util as Util

ascii_encoder = json.JSONEncoder(ensure_ascii=True)

class CompressorError(Exception):
    def __init__(self, message, node):
        Exception.__init__(self, "Compressor Error: %s for node type=%s in %s at line %s!" % (message, node.type, node.getFileName(), node.line))

class Compressor:
    __useIndenting = False
    __useBlockBreaks = False
    __useStatementBreaks = False
    __useWhiteSpace = False

    __indentLevel = 0

    __simple = ["true", "false", "null"]

    __dividers = {
        "plus"        : '+',
        "minus"       : '-',
        "mul"         : '*',
        "div"         : '/',
        "mod"         : '%',
        "dot"         : '.',
        "or"          : "||",
        "and"         : "&&",
        "strict_eq"   : '===',
        "eq"          : '==',
        "strict_ne"   : '!==',
        "ne"          : '!=',
        "lsh"         : '<<',
        "le"          : '<=',
        "lt"          : '<',
        "ursh"        : '>>>',
        "rsh"         : '>>',
        "ge"          : '>=',
        "gt"          : '>',
        "bitwise_or"  : '|',
        "bitwise_xor" : '^',
        "bitwise_and" : '&'
    }

    __prefixes = {
        "increment"   : "++",
        "decrement"   : "--",
        "bitwise_not" : '~',
        "not"         : "!",
        "unary_plus"  : "+",
        "unary_minus" : "-",
        "delete"      : "delete ",
        "new"         : "new ",
        "typeof"      : "typeof ",
        "void"        : "void "
    }

    def __init__(self, format=None):
        if format:
            if format.has("indent"):
                self.__useIndenting = True

            if format.has("blocks"):
                self.__useBlockBreaks = True

            if format.has("statements"):
                self.__useStatementBreaks = True

            if format.has("whitespace"):
                self.__useWhiteSpace = True

    def compress(self, node):
        """
        Compresses the given node and returns the compressed text result
        """

        type = node.type

        if type in self.__simple:
            result = type
        elif type in self.__prefixes:
            if getattr(node, "postfix", False):
                result = self.compress(node[0]) + self.__prefixes[node.type]
            else:
                result = self.__prefixes[node.type] + self.compress(node[0])

        elif type in self.__dividers:
            first = self.compress(node[0])
            second = self.compress(node[1])
            divider = self.__dividers[node.type]

            # Fast path
            if node.type not in ("plus", "minus"):
                result = "%s%s%s" % (first, divider, second)

            # Special code for dealing with situations like x + ++y and y-- - x
            else:
                result = first
                if first.endswith(divider):
                    result += " "

                result += divider

                if second.startswith(divider):
                    result += " "

                result += second

        else:
            try:
                result = getattr(self, "type_%s" % type)(node)
            except AttributeError:
                raise Exception("Style compressor does not support type '%s' from line %s in file %s" % (type, node.line, node.getFileName()))

        return result

    def indent(self, code):
        """
        Indents the given code by the current indenting setup
        """

        if not self.__useIndenting:
            return code

        lines = code.split("\n")
        result = []
        prefix = self.__indentLevel * "  "

        for line in lines:
            if line:
                result.append("%s%s" % (prefix, line))

        return "\n".join(result)

    #
    # Sheet Scope
    #

    def type_sheet(self, node):
        return self.__statements(node)

    def type_selector(self, node):
        # Ignore selectors without rules
        if len(node.rules) == 0:
            return ""

        selector = node.name

        if self.__useBlockBreaks:
            result = ",\n".join(selector)
        elif self.__useWhiteSpace:
            result = ", ".join(selector)
        else:
            result = ",".join(selector)

        return self.indent("%s%s" % (result, self.compress(node.rules)))

    def type_string(self, node):
        if hasattr(node, "quote"):
            return "%s%s%s" % (node.quote, node.value, node.quote)
        else:
            return ascii_encoder.encode(node.value)

    def type_number(self, node):
        value = node.value

        # Apply basic rounding to reduce size overhead of floating point
        if type(value) is float and not hasattr(node, "precision"):
            value = round(value, 4)

        # Special handling for protected float/exponential
        if type(value) == str:
            # Convert zero-prefix
            if value.startswith("0.") and len(value) > 2:
                value = value[1:]

            # Convert zero postfix
            elif value.endswith(".0"):
                value = value[:-2]

        elif int(value) == value and node.parent.type != "dot":
            value = int(value)

        return "%s%s" % (value, getattr(node, "unit", ""))

    def type_identifier(self, node):
        return node.value

    def type_slash(self, node):
        return "/"

    def type_expr(self, node):
        child = node[0]
        return self.compress(child)

    def type_raw(self, node):
        child = node[0]
        result = self.compress(child)

        # Remove string quotes
        if child.type == "string":
            result = result[1:-1]

        return result

    def type_list(self, node):
        return " ".join([ self.compress(child) for child in node ])

    def type_comma(self, node):
        first = self.compress(node[0])
        second = self.compress(node[1])
        if self.__useWhiteSpace:
            res = "%s, %s"
        else:
            res = "%s,%s"

        return res % (first, second)

    def type_block(self, node):
        self.__indentLevel += 1
        inner = self.__statements(node)
        self.__indentLevel -= 1

        if self.__useBlockBreaks:
            return "{\n%s\n}\n" % inner
        else:
            return "{%s}" % inner

    def type_property(self, node):
        self.__indentLevel += 1
        inner = self.__values(node)
        self.__indentLevel -= 1

        if self.__useWhiteSpace:
            return self.indent("%s: %s;" % (node.name, inner))
        else:
            return self.indent("%s:%s;" % (node.name, inner))

    def type_call(self, node):
        params = []
        callParams = getattr(node, "params", None)
        if callParams:
            for paramNode in callParams:
                params.append(self.compress(paramNode))

        if self.__useWhiteSpace:
            paramsJoin = ", "
        else:
            paramsJoin = ","

        return "%s(%s)" % (node[0].value, paramsJoin.join(params))

    def type_mixin(self, node):
        # Filter out non-extend mixins
        if not hasattr(node, "selector"):
            raise CompressorError("Left mixin \"%s\" found in tree to compress" % node.name, node)

        selector = node.selector

        if self.__useBlockBreaks:
            result = ",\n".join(selector)
        elif self.__useWhiteSpace:
            result = ", ".join(selector)
        else:
            result = ",".join(selector)

        self.__indentLevel += 1
        inner = self.__statements(node.rules)
        self.__indentLevel -= 1

        if self.__useBlockBreaks:
            result += "{\n%s\n}" % inner
        else:
            result += "{%s}" % inner

        return result

    def type_system(self, node):
        name = node.name

        if not Util.isBuiltin(name):
            raise CompressorError("Unsupported built-in method: %s" % name, node)

        if self.__useWhiteSpace:
            separator = ", "
        else:
            separator = ","

        return "%s(%s)" % (name, separator.join([ self.compress(child) for child in node.params ]))

    def type_keyframes(self, node):
        vendor = getattr(node, "vendor", None)
        if vendor:
            result = "@-%s-keyframes " % vendor
        else:
            result = "@keyframes "

        result += "%s{" % node.name

        if self.__useBlockBreaks:
            result += "\n"

        self.__indentLevel += 1

        frames = "".join([ self.compress(child) for child in node ])
        result += self.indent(frames)

        self.__indentLevel -= 1

        if self.__useBlockBreaks:
            result += "\n"

        result += "}"

        return result

    def type_frame(self, node):
        result = ""

        if self.__useWhiteSpace:
            result += node.value.replace(",", ", ")
        else:
            result += node.value

        result += self.compress(node.rules)

        return result

    def type_media(self, node):
        if self.__useBlockBreaks:
            separator = ",\n"
        elif self.__useWhiteSpace:
            separator = ", "
        else:
            separator = ","

        result = "@media %s" % separator.join(node.name)
        result += self.compress(node.rules)

        return self.indent(result)

    def type_fontface(self, node):
        result = "@font-face" + self.compress(node.rules)

        return self.indent(result)

    #
    # Helpers
    #

    def __statements(self, node):
        result = []
        for child in node:
            result.append(self.compress(child))

        if self.__useStatementBreaks:
            return "\n".join(result)
        else:
            return "".join(result)

    def __values(self, node):
        result = []
        for child in node:
            result.append(str(self.compress(child)))

        return " ".join(result)

<commit_msg>Ignore direct blocks inside blocks<commit_after>#
# Jasy - Web Tooling Framework
# Copyright 2013 Sebastian Werner
#

import json, re, sys
import jasy.style.Util as Util

ascii_encoder = json.JSONEncoder(ensure_ascii=True)

class CompressorError(Exception):
    def __init__(self, message, node):
        Exception.__init__(self, "Compressor Error: %s for node type=%s in %s at line %s!" % (message, node.type, node.getFileName(), node.line))

class Compressor:
    __useIndenting = False
    __useBlockBreaks = False
    __useStatementBreaks = False
    __useWhiteSpace = False

    __indentLevel = 0

    __simple = ["true", "false", "null"]

    __dividers = {
        "plus"        : '+',
        "minus"       : '-',
        "mul"         : '*',
        "div"         : '/',
        "mod"         : '%',
        "dot"         : '.',
        "or"          : "||",
        "and"         : "&&",
        "strict_eq"   : '===',
        "eq"          : '==',
        "strict_ne"   : '!==',
        "ne"          : '!=',
        "lsh"         : '<<',
        "le"          : '<=',
        "lt"          : '<',
        "ursh"        : '>>>',
        "rsh"         : '>>',
        "ge"          : '>=',
        "gt"          : '>',
        "bitwise_or"  : '|',
        "bitwise_xor" : '^',
        "bitwise_and" : '&'
    }

    __prefixes = {
        "increment"   : "++",
        "decrement"   : "--",
        "bitwise_not" : '~',
        "not"         : "!",
        "unary_plus"  : "+",
        "unary_minus" : "-",
        "delete"      : "delete ",
        "new"         : "new ",
        "typeof"      : "typeof ",
        "void"        : "void "
    }

    def __init__(self, format=None):
        if format:
            if format.has("indent"):
                self.__useIndenting = True

            if format.has("blocks"):
                self.__useBlockBreaks = True

            if format.has("statements"):
                self.__useStatementBreaks = True

            if format.has("whitespace"):
                self.__useWhiteSpace = True

    def compress(self, node):
        """
        Compresses the given node and returns the compressed text result
        """

        type = node.type

        if type in self.__simple:
            result = type
        elif type in self.__prefixes:
            if getattr(node, "postfix", False):
                result = self.compress(node[0]) + self.__prefixes[node.type]
            else:
                result = self.__prefixes[node.type] + self.compress(node[0])

        elif type in self.__dividers:
            first = self.compress(node[0])
            second = self.compress(node[1])
            divider = self.__dividers[node.type]

            # Fast path
            if node.type not in ("plus", "minus"):
                result = "%s%s%s" % (first, divider, second)

            # Special code for dealing with situations like x + ++y and y-- - x
            else:
                result = first
                if first.endswith(divider):
                    result += " "

                result += divider

                if second.startswith(divider):
                    result += " "

                result += second

        else:
            try:
                result = getattr(self, "type_%s" % type)(node)
            except AttributeError:
                raise Exception("Style compressor does not support type '%s' from line %s in file %s" % (type, node.line, node.getFileName()))

        return result

    def indent(self, code):
        """
        Indents the given code by the current indenting setup
        """

        if not self.__useIndenting:
            return code

        lines = code.split("\n")
        result = []
        prefix = self.__indentLevel * "  "

        for line in lines:
            if line:
                result.append("%s%s" % (prefix, line))

        return "\n".join(result)

    #
    # Sheet Scope
    #

    def type_sheet(self, node):
        return self.__statements(node)

    def type_selector(self, node):
        # Ignore selectors without rules
        if len(node.rules) == 0:
            return ""

        selector = node.name

        if self.__useBlockBreaks:
            result = ",\n".join(selector)
        elif self.__useWhiteSpace:
            result = ", ".join(selector)
        else:
            result = ",".join(selector)

        return self.indent("%s%s" % (result, self.compress(node.rules)))

    def type_string(self, node):
        if hasattr(node, "quote"):
            return "%s%s%s" % (node.quote, node.value, node.quote)
        else:
            return ascii_encoder.encode(node.value)

    def type_number(self, node):
        value = node.value

        # Apply basic rounding to reduce size overhead of floating point
        if type(value) is float and not hasattr(node, "precision"):
            value = round(value, 4)

        # Special handling for protected float/exponential
        if type(value) == str:
            # Convert zero-prefix
            if value.startswith("0.") and len(value) > 2:
                value = value[1:]

            # Convert zero postfix
            elif value.endswith(".0"):
                value = value[:-2]

        elif int(value) == value and node.parent.type != "dot":
            value = int(value)

        return "%s%s" % (value, getattr(node, "unit", ""))

    def type_identifier(self, node):
        return node.value

    def type_slash(self, node):
        return "/"

    def type_expr(self, node):
        child = node[0]
        return self.compress(child)

    def type_raw(self, node):
        child = node[0]
        result = self.compress(child)

        # Remove string quotes
        if child.type == "string":
            result = result[1:-1]

        return result

    def type_list(self, node):
        return " ".join([ self.compress(child) for child in node ])

    def type_comma(self, node):
        first = self.compress(node[0])
        second = self.compress(node[1])
        if self.__useWhiteSpace:
            res = "%s, %s"
        else:
            res = "%s,%s"

        return res % (first, second)

    def type_block(self, node):
        # Ignore direct blocks inside blocks
        if node.parent.type == "block":
            return self.__statements(node)

        self.__indentLevel += 1
        inner = self.__statements(node)
        self.__indentLevel -= 1

        if self.__useBlockBreaks:
            return "{\n%s\n}\n" % inner
        else:
            return "{%s}" % inner

    def type_property(self, node):
        self.__indentLevel += 1
        inner = self.__values(node)
        self.__indentLevel -= 1

        if self.__useWhiteSpace:
            return self.indent("%s: %s;" % (node.name, inner))
        else:
            return self.indent("%s:%s;" % (node.name, inner))

    def type_call(self, node):
        params = []
        callParams = getattr(node, "params", None)
        if callParams:
            for paramNode in callParams:
                params.append(self.compress(paramNode))

        if self.__useWhiteSpace:
            paramsJoin = ", "
        else:
            paramsJoin = ","

        return "%s(%s)" % (node[0].value, paramsJoin.join(params))

    def type_mixin(self, node):
        # Filter out non-extend mixins
        if not hasattr(node, "selector"):
            raise CompressorError("Left mixin \"%s\" found in tree to compress" % node.name, node)

        selector = node.selector

        if self.__useBlockBreaks:
            result = ",\n".join(selector)
        elif self.__useWhiteSpace:
            result = ", ".join(selector)
        else:
            result = ",".join(selector)

        self.__indentLevel += 1
        inner = self.__statements(node.rules)
        self.__indentLevel -= 1

        if self.__useBlockBreaks:
            result += "{\n%s\n}" % inner
        else:
            result += "{%s}" % inner

        return result

    def type_system(self, node):
        name = node.name

        if not Util.isBuiltin(name):
            raise CompressorError("Unsupported built-in method: %s" % name, node)

        if self.__useWhiteSpace:
            separator = ", "
        else:
            separator = ","

        return "%s(%s)" % (name, separator.join([ self.compress(child) for child in node.params ]))

    def type_keyframes(self, node):
        vendor = getattr(node, "vendor", None)
        if vendor:
            result = "@-%s-keyframes " % vendor
        else:
            result = "@keyframes "

        result += "%s{" % node.name

        if self.__useBlockBreaks:
            result += "\n"

        self.__indentLevel += 1

        frames = "".join([ self.compress(child) for child in node ])
        result += self.indent(frames)

        self.__indentLevel -= 1

        if self.__useBlockBreaks:
            result += "\n"

        result += "}"

        return result

    def type_frame(self, node):
        result = ""

        if self.__useWhiteSpace:
            result += node.value.replace(",", ", ")
        else:
            result += node.value

        result += self.compress(node.rules)

        return result

    def type_media(self, node):
        if self.__useBlockBreaks:
            separator = ",\n"
        elif self.__useWhiteSpace:
            separator = ", "
        else:
            separator = ","

        result = "@media %s" % separator.join(node.name)
        result += self.compress(node.rules)

        return self.indent(result)

    def type_fontface(self, node):
        result = "@font-face" + self.compress(node.rules)

        return self.indent(result)

    #
    # Helpers
    #

    def __statements(self, node):
        result = []
        for child in node:
            result.append(self.compress(child))

        if self.__useStatementBreaks:
            return "\n".join(result)
        else:
            return "".join(result)

    def __values(self, node):
        result = []
        for child in node:
            result.append(str(self.compress(child)))

        return " ".join(result)

<|endoftext|>