<commit_before># Copyright (c) 2015 Intracom S.A. Telecom Solutions. All rights reserved.
#
# This program and the accompanying materials are made available under the
# terms of the Eclipse Public License v1.0 which accompanies this distribution,
# and is available at http://www.eclipse.org/legal/epl-v10.html

""" NSTAT NorthBound  """

import emulators.nb_generator.flow_utils
import json
import logging
import multinet_utils
import multiprocessing
import re
import time
import util.netutil

def nb_generator_start(nb_generator_ssh_client,nb_generator_base_dir,nb_generator_cpus,
                       nb_generator_handlers_set,controller_node,
                       controller_nb_interface,total_flows,flow_workers,
                       flow_operations_delay_ms,flow_delete_flag,log_level):
    cmd = ('cd {0}; taskset -c {1} python3.4 {2} {3} {4} {5} {6} {7} {8} {9} {10} {11}'.
           format(nb_generator_base_dir, nb_generator_cpus,
                  nb_generator_handlers_set.run_handler,
                  controller_node.ip, controller_nb_interface.port,
                  total_flows, flow_workers, flow_operations_delay_ms,
                  flow_delete_flag, controller_nb_interface.username,
                  controller_nb_interface.password, log_level))
    logging.debug('Generator handler command:{0}.'.format(cmd))

    exit_status , output = util.netutil.ssh_run_command(nb_generator_ssh_client,
                                                        cmd,
                                                        '[generator_run_handler]')

    if exit_status!=0:
        raise Exception('northbound generator failed')

    return output

def poll_flows_dastastore(result_queue, expected_flows, t_start, controller_nb_interface):
    """
    Monitors operational DS until the expected number of flows are found or the
    deadline is reached.

    :param expected_flows: expected number of flows
    :param ctrl_ip: controller IP
    :param ctrl_port: controller RESTconf port
    :param t_start: timestamp for begin of discovery
    :param auth_token: RESTconf authorization token (username/password tuple)

    :returns: Returns a float number containing the time in which
    expected_flows were discovered otherwise containing -1.0 on failure.
    flow_transmission_start
    :rtype: float
    :type expected_flows: int
    :type ctrl_ip: str
    :type ctrl_port: int
    :type t_start: float
    :type auth_token: tuple<str>
    """

    deadline = 240
    odl_inventory = emulators.nb_generator.flow_utils.FlowExplorer(controller_nb_interface.ip,
                                                         controller_nb_interface.port,
                                                         'operational',
                                                         (controller_nb_interface.username, controller_nb_interface.password))
    t_discovery_start = time.time()
    previous_discovered_flows = 0

    while True:
        if (time.time() - t_discovery_start) > deadline:
            logging.info('[flow_master_thread] Deadline of {0} seconds '
                         'passed'.format(deadline))
            result_queue.put({'end_to_end_flows_operation_time': -1.0}, block=True)
            return
        else:
            odl_inventory.get_inventory_flows_stats()
            logging.debug('Found {0} flows at inventory'.
                          format(odl_inventory.found_flows))
            if (odl_inventory.found_flows - previous_discovered_flows) != 0:
                t_discovery_start = time.time()
                previous_discovered_flows = odl_inventory.found_flows
            if odl_inventory.found_flows == expected_flows:
                time_interval = time.time() - t_start
                logging.debug('[flow_master_thread] Flow-Master '
                             '{0} flows found in {1} seconds'.
                             format(expected_flows, time_interval))

                result_queue.put({'end_to_end_flows_operation_time': time_interval}, block=True)

                return

        time.sleep(1)

def poll_flows_dastastore_confirm(result_queue, expected_flows, controller_nb_interface):
    """
    Monitors operational DS until the expected number of flows are found or the
    deadline is reached.

    :param expected_flows: expected number of flows
    :param ctrl_ip: controller IP
    :param ctrl_port: controller RESTconf port
    :param t_start: timestamp for begin of discovery
    :param auth_token: RESTconf authorization token (username/password tuple)

    :returns: Returns a float number containing the time in which
    expected_flows were discovered otherwise containing -1.0 on failure.
    flow_transmission_start
    :rtype: float
    :type expected_flows: int
    :type ctrl_ip: str
    :type ctrl_port: int
    :type t_start: float
    :type auth_token: tuple<str>
    """

    deadline = 240
    odl_inventory = emulators.nb_generator.flow_utils.FlowExplorer(controller_nb_interface.ip,
                                                         controller_nb_interface.port,
                                                         'operational',
                                                         (controller_nb_interface.username, controller_nb_interface.password))
    t_discovery_start = time.time()
    t_start = time.time()
    previous_discovered_flows = 0

    while True:
        if (time.time() - t_discovery_start) > deadline:
            logging.info('[flow_master_thread] Deadline of {0} seconds '
                         'passed'.format(deadline))
            result_queue.put({'confirm_time': -1.0}, block=True)
            return
        else:
            odl_inventory.get_inventory_flows_stats()
            logging.debug('Found {0} flows at inventory'.
                          format(odl_inventory.found_flows))
            if (odl_inventory.found_flows - previous_discovered_flows) != 0:
                t_discovery_start = time.time()
                previous_discovered_flows = odl_inventory.found_flows
            if odl_inventory.found_flows == expected_flows:
                time_interval = time.time() - t_start
                logging.debug('[flow_master_thread] Flow-Master '
                             '{0} flows found in {1} seconds'.
                             format(expected_flows, time_interval))

                result_queue.put({'confirm_time': time_interval}, block=True)

                return

        time.sleep(1)

def poll_flows_switches(result_queue, expected_flows, t_start,get_flows_handler,multinet_base_dir):
    deadline = 240
    t_discovery_start = time.time()
    previous_discovered_flows = 0

    while True:
        if (time.time() - t_discovery_start) > deadline:
            logging.info('[flow_master_thread] Deadline of {0} seconds '
                         'passed'.format(deadline))
            result_queue.put({'switch_operation_time': -1.0}, block=True)
            return
        else:
            result_get_flows = multinet_utils.multinet_command_runner(get_flows_handler,
                '[get_flows_handler]', multinet_base_dir,
                is_privileged=False)

            regex_result = re.search(r'INFO:root:\[get_flows_topology_handler\]\[response data\].*', result_get_flows)
            if regex_result == None:
                raise Exception('Failed to get number of flows from network switches')
            else:
                result_get_flows = regex_result.group(0).replace('INFO:root:[get_flows_topology_handler][response data] ', '')
            discovered_flows = sum([list(json.loads(v).values())[0] for v in json.loads(result_get_flows)])

            logging.debug('Found {0} flows at topology switches'.
                          format(discovered_flows))
            if (discovered_flows - previous_discovered_flows) != 0:
                t_discovery_start = time.time()
                previous_discovered_flows = discovered_flows
            if discovered_flows == expected_flows:
                time_interval = time.time() - t_start
                logging.debug('expected flows = {0} \n '
                             'discovered flows = {1}'.
                             format(expected_flows, discovered_flows))

                result_queue.put({'switch_operation_time': time_interval}, block=True)

                return

        time.sleep(1)
    return

def monitor_threads_run(expected_flows, t_start, controller_nb_interface,
                        get_flows_handler,multinet_base_dir):

    logging.info('creating result queues for ...')
    result_queue = multiprocessing.Queue()
    logging.info('creating thread for end_to_end_installation_time measurement')
    monitor_thread_ds = multiprocessing.Process(target=poll_flows_dastastore,
                                             args=(result_queue,expected_flows,
                                                  t_start,
                                                  controller_nb_interface))
    monitor_thread_sw = multiprocessing.Process(target=poll_flows_switches,
                                             args=(result_queue,expected_flows,
                                                  t_start,get_flows_handler,multinet_base_dir))
    monitor_thread_ds_confirm = multiprocessing.Process(target=poll_flows_dastastore_confirm,
                                             args=(result_queue,expected_flows,
                                                   controller_nb_interface))
    monitor_thread_ds.start()
    monitor_thread_sw.start()
    monitor_thread_ds_confirm.start()

    monitor_thread_ds.join()
    monitor_thread_sw.join()
    monitor_thread_ds_confirm.join()
    results = {}
    while not result_queue.empty():
        results.update(result_queue.get())

    return results

<commit_msg>Refactor thread function to use multinet console output parser<commit_after># Copyright (c) 2015 Intracom S.A. Telecom Solutions. All rights reserved.
#
# This program and the accompanying materials are made available under the
# terms of the Eclipse Public License v1.0 which accompanies this distribution,
# and is available at http://www.eclipse.org/legal/epl-v10.html

""" NSTAT NorthBound  """

import emulators.nb_generator.flow_utils
import logging
import multinet_utils
import multiprocessing
import time
import util.netutil

def nb_generator_start(nb_generator_ssh_client,nb_generator_base_dir,nb_generator_cpus,
                       nb_generator_handlers_set,controller_node,
                       controller_nb_interface,total_flows,flow_workers,
                       flow_operations_delay_ms,flow_delete_flag,log_level):
    cmd = ('cd {0}; taskset -c {1} python3.4 {2} {3} {4} {5} {6} {7} {8} {9} {10} {11}'.
           format(nb_generator_base_dir, nb_generator_cpus,
                  nb_generator_handlers_set.run_handler,
                  controller_node.ip, controller_nb_interface.port,
                  total_flows, flow_workers, flow_operations_delay_ms,
                  flow_delete_flag, controller_nb_interface.username,
                  controller_nb_interface.password, log_level))
    logging.debug('Generator handler command:{0}.'.format(cmd))

    exit_status , output = util.netutil.ssh_run_command(nb_generator_ssh_client,
                                                        cmd,
                                                        '[generator_run_handler]')

    if exit_status!=0:
        raise Exception('northbound generator failed')

    return output

def poll_flows_dastastore(result_queue, expected_flows, t_start, controller_nb_interface):
    """
    Monitors operational DS until the expected number of flows are found or the
    deadline is reached.

    :param expected_flows: expected number of flows
    :param ctrl_ip: controller IP
    :param ctrl_port: controller RESTconf port
    :param t_start: timestamp for begin of discovery
    :param auth_token: RESTconf authorization token (username/password tuple)

    :returns: Returns a float number containing the time in which
    expected_flows were discovered otherwise containing -1.0 on failure.
    flow_transmission_start
    :rtype: float
    :type expected_flows: int
    :type ctrl_ip: str
    :type ctrl_port: int
    :type t_start: float
    :type auth_token: tuple<str>
    """

    deadline = 240
    odl_inventory = emulators.nb_generator.flow_utils.FlowExplorer(controller_nb_interface.ip,
                                                         controller_nb_interface.port,
                                                         'operational',
                                                         (controller_nb_interface.username, controller_nb_interface.password))
    t_discovery_start = time.time()
    previous_discovered_flows = 0

    while True:
        if (time.time() - t_discovery_start) > deadline:
            logging.info('[flow_master_thread] Deadline of {0} seconds '
                         'passed'.format(deadline))
            result_queue.put({'end_to_end_flows_operation_time': -1.0}, block=True)
            return
        else:
            odl_inventory.get_inventory_flows_stats()
            logging.debug('Found {0} flows at inventory'.
                          format(odl_inventory.found_flows))
            if (odl_inventory.found_flows - previous_discovered_flows) != 0:
                t_discovery_start = time.time()
                previous_discovered_flows = odl_inventory.found_flows
            if odl_inventory.found_flows == expected_flows:
                time_interval = time.time() - t_start
                logging.debug('[flow_master_thread] Flow-Master '
                             '{0} flows found in {1} seconds'.
                             format(expected_flows, time_interval))

                result_queue.put({'end_to_end_flows_operation_time': time_interval}, block=True)

                return

        time.sleep(1)

def poll_flows_dastastore_confirm(result_queue, expected_flows, controller_nb_interface):
    """
    Monitors operational DS until the expected number of flows are found or the
    deadline is reached.

    :param expected_flows: expected number of flows
    :param ctrl_ip: controller IP
    :param ctrl_port: controller RESTconf port
    :param t_start: timestamp for begin of discovery
    :param auth_token: RESTconf authorization token (username/password tuple)

    :returns: Returns a float number containing the time in which
    expected_flows were discovered otherwise containing -1.0 on failure.
    flow_transmission_start
    :rtype: float
    :type expected_flows: int
    :type ctrl_ip: str
    :type ctrl_port: int
    :type t_start: float
    :type auth_token: tuple<str>
    """

    deadline = 240
    odl_inventory = emulators.nb_generator.flow_utils.FlowExplorer(controller_nb_interface.ip,
                                                         controller_nb_interface.port,
                                                         'operational',
                                                         (controller_nb_interface.username, controller_nb_interface.password))
    t_discovery_start = time.time()
    t_start = time.time()
    previous_discovered_flows = 0

    while True:
        if (time.time() - t_discovery_start) > deadline:
            logging.info('[flow_master_thread] Deadline of {0} seconds '
                         'passed'.format(deadline))
            result_queue.put({'confirm_time': -1.0}, block=True)
            return
        else:
            odl_inventory.get_inventory_flows_stats()
            logging.debug('Found {0} flows at inventory'.
                          format(odl_inventory.found_flows))
            if (odl_inventory.found_flows - previous_discovered_flows) != 0:
                t_discovery_start = time.time()
                previous_discovered_flows = odl_inventory.found_flows
            if odl_inventory.found_flows == expected_flows:
                time_interval = time.time() - t_start
                logging.debug('[flow_master_thread] Flow-Master '
                             '{0} flows found in {1} seconds'.
                             format(expected_flows, time_interval))

                result_queue.put({'confirm_time': time_interval}, block=True)

                return

        time.sleep(1)

def poll_flows_switches(result_queue, expected_flows, t_start,get_flows_handler,multinet_base_dir):
    deadline = 240
    t_discovery_start = time.time()
    previous_discovered_flows = 0

    while True:
        if (time.time() - t_discovery_start) > deadline:
            logging.info('[flow_master_thread] Deadline of {0} seconds '
                         'passed'.format(deadline))
            result_queue.put({'switch_operation_time': -1.0}, block=True)
            return
        else:
            result_get_flows = multinet_utils.multinet_command_runner(get_flows_handler,
                '[get_flows_handler]', multinet_base_dir,
                is_privileged=False)
            # Get total flows from multinet topology switches
            discovered_flows = multinet_utils.parse_multinet_output('get_flows_topology_handler', result_get_flows)

            logging.debug('Found {0} flows at topology switches'.
                          format(discovered_flows))
            if (discovered_flows - previous_discovered_flows) != 0:
                t_discovery_start = time.time()
                previous_discovered_flows = discovered_flows
            if discovered_flows == expected_flows:
                time_interval = time.time() - t_start
                logging.debug('expected flows = {0} \n '
                             'discovered flows = {1}'.
                             format(expected_flows, discovered_flows))

                result_queue.put({'switch_operation_time': time_interval}, block=True)

                return

        time.sleep(1)
    return

def monitor_threads_run(expected_flows, t_start, controller_nb_interface,
                        get_flows_handler,multinet_base_dir):

    logging.info('creating result queues for ...')
    result_queue = multiprocessing.Queue()
    logging.info('creating thread for end_to_end_installation_time measurement')
    monitor_thread_ds = multiprocessing.Process(target=poll_flows_dastastore,
                                             args=(result_queue,expected_flows,
                                                  t_start,
                                                  controller_nb_interface))
    monitor_thread_sw = multiprocessing.Process(target=poll_flows_switches,
                                             args=(result_queue,expected_flows,
                                                  t_start,get_flows_handler,multinet_base_dir))
    monitor_thread_ds_confirm = multiprocessing.Process(target=poll_flows_dastastore_confirm,
                                             args=(result_queue,expected_flows,
                                                   controller_nb_interface))
    monitor_thread_ds.start()
    monitor_thread_sw.start()
    monitor_thread_ds_confirm.start()

    monitor_thread_ds.join()
    monitor_thread_sw.join()
    monitor_thread_ds_confirm.join()
    results = {}
    while not result_queue.empty():
        results.update(result_queue.get())

    return results

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import requests
import time

######################################
#      CONFIGURATION                 #
######################################
domain = "rationalwiki.org"
entry = "RationalWiki:Saloon_bar"
search_terms = ["nazi","ehrenstein","rape","arcane","dondrekhan"]
require = ""
exclude = " "
repeat_pages = False

#####################################

def web_r(url_frag):
	reqHeaders = {"User-Agent":"AKULA SOKOLOVSKAYA: sokolovskaya.akula.sturmkrieg.ru"}
	w_req = requests.get("http://"+ domain +""+ url_frag +"", headers=reqHeaders )
	return w_req

def get_links(response):
	page_links = []
	for a in response.find_all('a'):
		href = a.get('href', '')
		if href.startswith('/wiki') and exclude.lower() not in href.lower() and require.lower() in href.lower():
			page_links.append(href)
	return page_links

def msanititize(l_list):
	n_list = []
	for link in l_list:
		if link not in n_list:
			n_list.append(link)
	return n_list

req = web_r("/wiki/"+ entry)
res = req.text
res = BeautifulSoup(res)
links = get_links(res)
links = msanititize(links)
#print links
all_links = []
all_links.append(links)
alert_count = [0] * len(search_terms)
total_count = [0] * len(search_terms)
hasAlert = False
runs = 0

for cycle in all_links:
	links = all_links[runs]
	for link in links:
		if repeat_pages == False:
			for list in all_links:
				if link in list:
					print ""+ link +" was repeated and has been skipped."
					file = open("WikiCrawl_log.txt", 'a')
					file.write("\n"+ link +" was repeated and has been skipped.\n")
					file.close()
					continue
		req = web_r(link)
		res = req.text
		res = BeautifulSoup(res)
		for text in res.find_all("h1"):
			print "\n----------------------------------------------------------------------"+ text.get_text()
		links = get_links(res)
		links = msanititize(links)
		all_links.append(links)
		for text in res.find_all(['p']):
			print ""+ text.get_text() +"\n"
		for text in res.find_all(['p', "li", "span", "div"]):
			for term in search_terms:
				if term.lower() in text.get_text().lower():
					alert_count[search_terms.index(term)] = alert_count[search_terms.index(term)] + 1
					total_count[search_terms.index(term)] = total_count[search_terms.index(term)] + 1
		file = open("WikiCrawl_log.txt", 'a')
		file.write("\n\n"+ link +"\n")
		for term in search_terms:
			print ""+ term +" count = "+ str(alert_count[search_terms.index(term)])
			file.write(""+ term +" count = "+ str(alert_count[search_terms.index(term)]) +"\n")
		print "\n"
		file.write("\n")
		for term in search_terms:
			print "Total "+ term +" count = "+ str(total_count[search_terms.index(term)])
			file.write("Total "+ term +" count = "+ str(total_count[search_terms.index(term)]) +"\n")
		for text in res.find_all(['p',"li","span","div"]):
			for term in search_terms:
				if term.lower() in text.get_text().lower():
					hasAlert = True
		if hasAlert == True:
			alert_r = web_r(link +"?action=raw")
			alert_r = alert_r.content
			file.write(""+ alert_r +"\n")
		file.close()
		for i in alert_count:
			alert_count[alert_count.index(i)] = 0
		hasAlert = False
		time.sleep(1)
	runs = runs + 1
	time.sleep(1)

<commit_msg>Fixing logic error and improving memory management<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import requests
import time

######################################
#      CONFIGURATION                 #
######################################
domain = "rationalwiki.org"
entry = "RationalWiki:Saloon_bar"
search_terms = ["nazi","ehrenstein","rape","arcane","dondrekhan"]
require = ""
exclude = " "
repeat_pages = False

#####################################

def web_r(url_frag):
	reqHeaders = {"User-Agent":"AKULA SOKOLOVSKAYA: sokolovskaya.akula.sturmkrieg.ru"}
	w_req = requests.get("http://"+ domain +""+ url_frag +"", headers=reqHeaders )
	return w_req

def get_links(response):
	page_links = []
	for a in response.find_all('a'):
		href = a.get('href', '')
		if href.startswith('/wiki') and exclude.lower() not in href.lower() and require.lower() in href.lower():
			page_links.append(href)
	return page_links

def msanititize(l_list):
	n_list = []
	for link in l_list:
		if link not in n_list:
			n_list.append(link)
	return n_list

req = web_r("/wiki/"+ entry)
res = req.text
res = BeautifulSoup(res)
links = get_links(res)
links = msanititize(links)
#print links
all_links = []
all_links.append(links)
alert_count = [0] * len(search_terms)
total_count = [0] * len(search_terms)
hasAlert = False
runs = 0
crawled_links = []

for cycle in all_links:
	links = all_links[runs]
	for link in links:
		if repeat_pages == False:
			if link in crawled_links:
				print ""+ link +" was repeated and has been skipped."
				file = open("WikiCrawl_log.txt", 'a')
				file.write("\n"+ link +" was repeated and has been skipped.\n")
				file.close()
				continue
		req = web_r(link)
		res = req.text
		res = BeautifulSoup(res)
		for text in res.find_all("h1"):
			print "\n----------------------------------------------------------------------"+ text.get_text()
		links = get_links(res)
		links = msanititize(links)
		all_links.append(links)
		for text in res.find_all(['p']):
			print ""+ text.get_text() +"\n"
		for text in res.find_all(['p', "li", "span", "div"]):
			for term in search_terms:
				if term.lower() in text.get_text().lower():
					alert_count[search_terms.index(term)] = alert_count[search_terms.index(term)] + 1
					total_count[search_terms.index(term)] = total_count[search_terms.index(term)] + 1
		file = open("WikiCrawl_log.txt", 'a')
		file.write("\n\n"+ link +"\n")
		for term in search_terms:
			print ""+ term +" count = "+ str(alert_count[search_terms.index(term)])
			file.write(""+ term +" count = "+ str(alert_count[search_terms.index(term)]) +"\n")
		print "\n"
		file.write("\n")
		for term in search_terms:
			print "Total "+ term +" count = "+ str(total_count[search_terms.index(term)])
			file.write("Total "+ term +" count = "+ str(total_count[search_terms.index(term)]) +"\n")
		for text in res.find_all(['p',"li","span","div"]):
			for term in search_terms:
				if term.lower() in text.get_text().lower():
					hasAlert = True
		if hasAlert == True:
			alert_r = web_r(link +"?action=raw")
			alert_r = alert_r.content
			file.write(""+ alert_r +"\n")
		file.close()
		for i in alert_count:
			alert_count[alert_count.index(i)] = 0
		hasAlert = False
		if repeat_pages == False:
			for c_link in crawled_links:
				for l_list in all_links:
					if c_link in l_list:
						l_list.remove(l_list[l_list.index(c_link)])
		crawled_links.append(link)
		#time.sleep(0.25)
	runs = runs + 1
	time.sleep(0.25)
print "\n\nNo more links recieved.  Program terminated."

<|endoftext|>


<commit_before># Copyright 2015 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

class ExitHandlerContext(object):
    """The context which is passed to an exit handler function."""

    def __init__(self, launch_state, task_state):
        self.launch_state = launch_state
        self.task_state = task_state

def default_exit_handler(context, ignore_returncode=False):
    """
    Trigger teardown of launch.

    Use the returncode of the task for the launch if the launch was not already tearing down.

    @param ignore_returncode: If True, then ignore process return code.
    """
    # trigger tear down if not already tearing down
    if not context.launch_state.teardown:
        context.launch_state.teardown = True

    # set launch return code if not already set
    if (
        not context.launch_state.returncode and
        # since Python < 3.5 on Windows does not support signaling SIGINT to the subprocesses
        # we can't expect them to shutdown cleanly, therefore we ignore their return code
        (os.name != 'nt' or not context.launch_state.teardown) and
        # allow force ignoring
        not ignore_returncode
    ):
        try:
            rc = int(context.task_state.returncode)
        except (TypeError, ValueError):
            rc = 1 if bool(context.task_state.returncode) else 0
        context.launch_state.returncode = rc

def ignore_exit_handler(context):
    """Continue the launch and don't affect the returncode of the launch."""
    pass

def restart_exit_handler(context):
    """Request restart of the task."""
    context.task_state.restart = True

def primary_exit_handler(context):
    """
    Trigger teardown of launch and if teardown already in place set non-zero return code.

    Same as default exit handler but if teardown was triggered by another task
    ensure that the returncode is non-zero.
    """
    if context.launch_state.teardown:
        if not context.launch_state.returncode:
            context.launch_state.returncode = 1

    default_exit_handler(context)

def primary_ignore_returncode_exit_handler(context):
    """
    Trigger teardown of launch and ignore return codes.

    Same as primary exit handler but ignore return codes from teardown.
    """
    if context.launch_state.teardown:
        if not context.launch_state.returncode:
            context.launch_state.returncode = 1

    default_exit_handler(context, ignore_returncode=True)

def ignore_signal_exit_handler(context):
    """
    Succeed if the process received a shutdown signal on teardown.

    Ignores return code if launch sent a SIGINT or SIGKILL to the task.
    """
    if context.launch_state.teardown:
        # Check the return code
        if context.task_state.signals_received:
            context.task_state.returncode = 0

    default_exit_handler(context)

def exit_on_error_exit_handler(context):
    """
    If the return code indicates an error trigger a tear down.

    If the return code is zero continue the other tasks.
    """
    try:
        rc = int(context.task_state.returncode)
    except (TypeError, ValueError):
        rc = 1 if bool(context.task_state.returncode) else 0
    if rc:
        # trigger tear down if not already tearing down
        if not context.launch_state.teardown:
            context.launch_state.teardown = True
        # set return code if not already set
        if not context.launch_state.returncode:
            context.launch_state.returncode = rc
<commit_msg>handle Windows case<commit_after># Copyright 2015 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

class ExitHandlerContext(object):
    """The context which is passed to an exit handler function."""

    def __init__(self, launch_state, task_state):
        self.launch_state = launch_state
        self.task_state = task_state

def default_exit_handler(context, ignore_returncode=False):
    """
    Trigger teardown of launch.

    Use the returncode of the task for the launch if the launch was not already tearing down.

    @param ignore_returncode: If True, then ignore process return code.
    """
    # trigger tear down if not already tearing down
    if not context.launch_state.teardown:
        context.launch_state.teardown = True

    # set launch return code if not already set
    if (
        not context.launch_state.returncode and
        # since Python < 3.5 on Windows does not support signaling SIGINT to the subprocesses
        # we can't expect them to shutdown cleanly, therefore we ignore their return code
        (os.name != 'nt' or not context.launch_state.teardown) and
        # allow force ignoring
        not ignore_returncode
    ):
        try:
            rc = int(context.task_state.returncode)
        except (TypeError, ValueError):
            rc = 1 if bool(context.task_state.returncode) else 0
        context.launch_state.returncode = rc

def ignore_exit_handler(context):
    """Continue the launch and don't affect the returncode of the launch."""
    pass

def restart_exit_handler(context):
    """Request restart of the task."""
    context.task_state.restart = True

def primary_exit_handler(context):
    """
    Trigger teardown of launch and if teardown already in place set non-zero return code.

    Same as default exit handler but if teardown was triggered by another task
    ensure that the returncode is non-zero.
    """
    if context.launch_state.teardown:
        if not context.launch_state.returncode:
            context.launch_state.returncode = 1

    default_exit_handler(context)

def primary_ignore_returncode_exit_handler(context):
    """
    Trigger teardown of launch and ignore return codes.

    Same as primary exit handler but ignore return codes from teardown.
    """
    if context.launch_state.teardown:
        if not context.launch_state.returncode:
            context.launch_state.returncode = 1

    default_exit_handler(context, ignore_returncode=True)

def ignore_signal_exit_handler(context):
    """
    Succeed if the process received a shutdown signal on teardown.

    Ignores return code if launch sent a SIGINT or SIGKILL to the task.
    """
    if context.launch_state.teardown:
        # Check the return code
        if context.task_state.signals_received:
            context.task_state.returncode = 0

    default_exit_handler(context)

def exit_on_error_exit_handler(context):
    """
    If the return code indicates an error trigger a tear down.

    If the return code is zero continue the other tasks.
    On Windows the task return code is ignored if launch sent a SIGINT or
    SIGKILL to the task.
    """
    if context.launch_state.teardown and context.task_state.signals_received \
            and os.name == 'nt':
        context.task_state.returncode = 0
        return

    try:
        rc = int(context.task_state.returncode)
    except (TypeError, ValueError):
        rc = 1 if bool(context.task_state.returncode) else 0
    if rc:
        # trigger tear down if not already tearing down
        if not context.launch_state.teardown:
            context.launch_state.teardown = True
        # set return code if not already set
        if not context.launch_state.returncode:
            context.launch_state.returncode = rc
<|endoftext|>


<commit_before>from ethereum.utils import sha3, privtoaddr, encode_hex, denoms

CLUSTER_NAME = 'raiden'

GENESIS_STUB = {
    'config': {
        'homesteadBlock': 1
    },
    'nonce': '0x0000000000000042',
    'mixhash': '0x0000000000000000000000000000000000000000000000000000000000000000',
    'difficulty': '0x4000',
    'coinbase': '0x0000000000000000000000000000000000000000',
    'timestamp': '0x00',
    'parentHash': '0x0000000000000000000000000000000000000000000000000000000000000000',
    'extraData': CLUSTER_NAME,
    'gasLimit': '0xfffffffff'
}

def generate_accounts(seeds):
    """Create private keys and addresses for all seeds.
    """
    return {
        seed: dict(
            privatekey=encode_hex(sha3(seed)),
            address=encode_hex(privtoaddr(sha3(seed)))
        ) for seed in seeds}

def mk_genesis(accounts, initial_alloc=denoms.ether * 100000000):
    """
    Create a genesis-block dict with allocation for all `accounts`.

    :param accounts: list of account addresses (hex)
    :param initial_alloc: the amount to allocate for the `accounts`
    :return: genesis dict
    """
    genesis = GENESIS_STUB.copy()
    genesis['alloc'] = {
        account: {
            'balance': str(initial_alloc)
        }
        for account in accounts
    }
    # add the one-privatekey account ("1" * 64) for convenience
    genesis['alloc']['19e7e376e7c213b7e7e7e46cc70a5dd086daff2a'] = dict(balance=str(initial_alloc))
    return genesis
<commit_msg>Use higher difficulty for higher blocktime #63<commit_after>from ethereum.utils import sha3, privtoaddr, encode_hex, denoms

CLUSTER_NAME = 'raiden'

GENESIS_STUB = {
    'config': {
        'homesteadBlock': 1
    },
    'nonce': '0x0000000000000042',
    'mixhash': '0x0000000000000000000000000000000000000000000000000000000000000000',
    'difficulty': '0x40000',
    'coinbase': '0x0000000000000000000000000000000000000000',
    'timestamp': '0x00',
    'parentHash': '0x0000000000000000000000000000000000000000000000000000000000000000',
    'extraData': CLUSTER_NAME,
    'gasLimit': '0xfffffffff'
}

def generate_accounts(seeds):
    """Create private keys and addresses for all seeds.
    """
    return {
        seed: dict(
            privatekey=encode_hex(sha3(seed)),
            address=encode_hex(privtoaddr(sha3(seed)))
        ) for seed in seeds}

def mk_genesis(accounts, initial_alloc=denoms.ether * 100000000):
    """
    Create a genesis-block dict with allocation for all `accounts`.

    :param accounts: list of account addresses (hex)
    :param initial_alloc: the amount to allocate for the `accounts`
    :return: genesis dict
    """
    genesis = GENESIS_STUB.copy()
    genesis['alloc'] = {
        account: {
            'balance': str(initial_alloc)
        }
        for account in accounts
    }
    # add the one-privatekey account ("1" * 64) for convenience
    genesis['alloc']['19e7e376e7c213b7e7e7e46cc70a5dd086daff2a'] = dict(balance=str(initial_alloc))
    return genesis
<|endoftext|>


<commit_before>#!/usr/bin/python2.4
# Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

'''Adaptation of the extern.tclib classes for our needs.
'''

import re
import types

from grit import exception
import grit.extern.tclib

def Identity(i):
  return i

class BaseMessage(object):
  '''Base class with methods shared by Message and Translation.
  '''

  def __init__(self, text='', placeholders=[], description='', meaning=''):
    self.parts = []
    self.placeholders = []
    self.description = description
    self.meaning = meaning
    self.dirty = True  # True if self.id is (or might be) wrong
    self.id = 0

    if text != '':
      if not placeholders or placeholders == []:
        self.AppendText(text)
      else:
        tag_map = {}
        for placeholder in placeholders:
          tag_map[placeholder.GetPresentation()] = [placeholder, 0]
        tag_re = '(' + '|'.join(tag_map.keys()) + ')'
        # This creates a regexp like '(TAG1|TAG2|TAG3)'
        chunked_text = re.split(tag_re, text)
        for chunk in chunked_text:
          if chunk: # ignore empty chunk
            if tag_map.has_key(chunk):
              self.AppendPlaceholder(tag_map[chunk][0])
              tag_map[chunk][1] += 1 # increase placeholder use count
            else:
              self.AppendText(chunk)
        for key in tag_map.keys():
          assert tag_map[key][1] != 0

  def GetRealContent(self, escaping_function=Identity):
    '''Returns the original content, i.e. what your application and users
    will see.

    Specify a function to escape each translateable bit, if you like.
    '''
    bits = []
    for item in self.parts:
      if isinstance(item, types.StringTypes):
        bits.append(escaping_function(item))
      else:
        bits.append(item.GetOriginal())
    return ''.join(bits)

  def GetPresentableContent(self):
    presentable_content = []
    for part in self.parts:
      if isinstance(part, Placeholder):
        presentable_content.append(part.GetPresentation())
      else:
        presentable_content.append(part)
    return ''.join(presentable_content)

  def AppendPlaceholder(self, placeholder):
    assert isinstance(placeholder, Placeholder)
    dup = False
    for other in self.GetPlaceholders():
      if other.presentation == placeholder.presentation:
        assert other.original == placeholder.original
        dup = True

    if not dup:
      self.placeholders.append(placeholder)
    self.parts.append(placeholder)
    self.dirty = True

  def AppendText(self, text):
    assert isinstance(text, types.StringTypes)
    assert text != ''

    self.parts.append(text)
    self.dirty = True

  def GetContent(self):
    '''Returns the parts of the message.  You may modify parts if you wish.
    Note that you must not call GetId() on this object until you have finished
    modifying the contents.
    '''
    self.dirty = True  # user might modify content
    return self.parts

  def GetDescription(self):
    return self.description

  def SetDescription(self, description):
    self.description = description

  def GetMeaning(self):
    return self.meaning

  def GetId(self):
    if self.dirty:
      self.id = self.GenerateId()
      self.dirty = False
    return self.id

  def GenerateId(self):
    # Must use a UTF-8 encoded version of the presentable content, along with
    # the meaning attribute, to match the TC.
    return grit.extern.tclib.GenerateMessageId(
      self.GetPresentableContent().encode('utf-8'), self.meaning)

  def GetPlaceholders(self):
    return self.placeholders

  def FillTclibBaseMessage(self, msg):
    msg.SetDescription(self.description.encode('utf-8'))

    for part in self.parts:
      if isinstance(part, Placeholder):
        ph = grit.extern.tclib.Placeholder(
          part.presentation.encode('utf-8'),
          part.original.encode('utf-8'),
          part.example.encode('utf-8'))
        msg.AppendPlaceholder(ph)
      else:
        msg.AppendText(part.encode('utf-8'))

class Message(BaseMessage):
  '''A message.'''

  def __init__(self, text='', placeholders=[], description='', meaning='',
               assigned_id=None):
    BaseMessage.__init__(self, text, placeholders, description, meaning)
    self.assigned_id = assigned_id

  def ToTclibMessage(self):
    msg = grit.extern.tclib.Message('utf-8', meaning=self.meaning)
    self.FillTclibBaseMessage(msg)
    return msg

  def GetId(self):
    '''Use the assigned id if we have one.'''
    if self.assigned_id:
      return self.assigned_id

    return BaseMessage.GetId(self)

class Translation(BaseMessage):
  '''A translation.'''

  def __init__(self, text='', id='', placeholders=[], description='', meaning=''):
    BaseMessage.__init__(self, text, placeholders, description, meaning)
    self.id = id

  def GetId(self):
    assert id != '', "ID has not been set."
    return self.id

  def SetId(self, id):
    self.id = id

  def ToTclibMessage(self):
    msg = grit.extern.tclib.Message(
      'utf-8', id=self.id, meaning=self.meaning)
    self.FillTclibBaseMessage(msg)
    return msg

class Placeholder(grit.extern.tclib.Placeholder):
  '''Modifies constructor to accept a Unicode string
  '''

  # Must match placeholder presentation names
  _NAME_RE = re.compile('[A-Za-z0-9_]+')

  def __init__(self, presentation, original, example):
    '''Creates a new placeholder.

    Args:
      presentation: 'USERNAME'
      original: '%s'
      example: 'Joi'
    '''
    assert presentation != ''
    assert original != ''
    assert example != ''
    if not self._NAME_RE.match(presentation):
      raise exception.InvalidPlaceholderName(presentation)
    self.presentation = presentation
    self.original = original
    self.example = example

  def GetPresentation(self):
    return self.presentation

  def GetOriginal(self):
    return self.original

  def GetExample(self):
    return self.example

<commit_msg>Enforce the restriction that placeholder names can only contain caps, numbers and underscores.<commit_after>#!/usr/bin/python2.4
# Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

'''Adaptation of the extern.tclib classes for our needs.
'''

import re
import types

from grit import exception
import grit.extern.tclib

def Identity(i):
  return i

class BaseMessage(object):
  '''Base class with methods shared by Message and Translation.
  '''

  def __init__(self, text='', placeholders=[], description='', meaning=''):
    self.parts = []
    self.placeholders = []
    self.description = description
    self.meaning = meaning
    self.dirty = True  # True if self.id is (or might be) wrong
    self.id = 0

    if text != '':
      if not placeholders or placeholders == []:
        self.AppendText(text)
      else:
        tag_map = {}
        for placeholder in placeholders:
          tag_map[placeholder.GetPresentation()] = [placeholder, 0]
        tag_re = '(' + '|'.join(tag_map.keys()) + ')'
        # This creates a regexp like '(TAG1|TAG2|TAG3)'
        chunked_text = re.split(tag_re, text)
        for chunk in chunked_text:
          if chunk: # ignore empty chunk
            if tag_map.has_key(chunk):
              self.AppendPlaceholder(tag_map[chunk][0])
              tag_map[chunk][1] += 1 # increase placeholder use count
            else:
              self.AppendText(chunk)
        for key in tag_map.keys():
          assert tag_map[key][1] != 0

  def GetRealContent(self, escaping_function=Identity):
    '''Returns the original content, i.e. what your application and users
    will see.

    Specify a function to escape each translateable bit, if you like.
    '''
    bits = []
    for item in self.parts:
      if isinstance(item, types.StringTypes):
        bits.append(escaping_function(item))
      else:
        bits.append(item.GetOriginal())
    return ''.join(bits)

  def GetPresentableContent(self):
    presentable_content = []
    for part in self.parts:
      if isinstance(part, Placeholder):
        presentable_content.append(part.GetPresentation())
      else:
        presentable_content.append(part)
    return ''.join(presentable_content)

  def AppendPlaceholder(self, placeholder):
    assert isinstance(placeholder, Placeholder)
    dup = False
    for other in self.GetPlaceholders():
      if other.presentation == placeholder.presentation:
        assert other.original == placeholder.original
        dup = True

    if not dup:
      self.placeholders.append(placeholder)
    self.parts.append(placeholder)
    self.dirty = True

  def AppendText(self, text):
    assert isinstance(text, types.StringTypes)
    assert text != ''

    self.parts.append(text)
    self.dirty = True

  def GetContent(self):
    '''Returns the parts of the message.  You may modify parts if you wish.
    Note that you must not call GetId() on this object until you have finished
    modifying the contents.
    '''
    self.dirty = True  # user might modify content
    return self.parts

  def GetDescription(self):
    return self.description

  def SetDescription(self, description):
    self.description = description

  def GetMeaning(self):
    return self.meaning

  def GetId(self):
    if self.dirty:
      self.id = self.GenerateId()
      self.dirty = False
    return self.id

  def GenerateId(self):
    # Must use a UTF-8 encoded version of the presentable content, along with
    # the meaning attribute, to match the TC.
    return grit.extern.tclib.GenerateMessageId(
      self.GetPresentableContent().encode('utf-8'), self.meaning)

  def GetPlaceholders(self):
    return self.placeholders

  def FillTclibBaseMessage(self, msg):
    msg.SetDescription(self.description.encode('utf-8'))

    for part in self.parts:
      if isinstance(part, Placeholder):
        ph = grit.extern.tclib.Placeholder(
          part.presentation.encode('utf-8'),
          part.original.encode('utf-8'),
          part.example.encode('utf-8'))
        msg.AppendPlaceholder(ph)
      else:
        msg.AppendText(part.encode('utf-8'))

class Message(BaseMessage):
  '''A message.'''

  def __init__(self, text='', placeholders=[], description='', meaning='',
               assigned_id=None):
    BaseMessage.__init__(self, text, placeholders, description, meaning)
    self.assigned_id = assigned_id

  def ToTclibMessage(self):
    msg = grit.extern.tclib.Message('utf-8', meaning=self.meaning)
    self.FillTclibBaseMessage(msg)
    return msg

  def GetId(self):
    '''Use the assigned id if we have one.'''
    if self.assigned_id:
      return self.assigned_id

    return BaseMessage.GetId(self)

class Translation(BaseMessage):
  '''A translation.'''

  def __init__(self, text='', id='', placeholders=[], description='', meaning=''):
    BaseMessage.__init__(self, text, placeholders, description, meaning)
    self.id = id

  def GetId(self):
    assert id != '', "ID has not been set."
    return self.id

  def SetId(self, id):
    self.id = id

  def ToTclibMessage(self):
    msg = grit.extern.tclib.Message(
      'utf-8', id=self.id, meaning=self.meaning)
    self.FillTclibBaseMessage(msg)
    return msg

class Placeholder(grit.extern.tclib.Placeholder):
  '''Modifies constructor to accept a Unicode string
  '''

  # Must match placeholder presentation names
  _NAME_RE = re.compile('^[A-Za-z0-9_]+$')

  def __init__(self, presentation, original, example):
    '''Creates a new placeholder.

    Args:
      presentation: 'USERNAME'
      original: '%s'
      example: 'Joi'
    '''
    assert presentation != ''
    assert original != ''
    assert example != ''
    if not self._NAME_RE.match(presentation):
      raise exception.InvalidPlaceholderName(presentation)
    self.presentation = presentation
    self.original = original
    self.example = example

  def GetPresentation(self):
    return self.presentation

  def GetOriginal(self):
    return self.original

  def GetExample(self):
    return self.example

<|endoftext|>


<commit_before>#
# (re)generate unicode property and type databases
#
# this script converts a unicode 3.2 database file to
# Modules/unicodedata_db.h, Modules/unicodename_db.h,
# and Objects/unicodetype_db.h
#
# history:
# 2000-09-24 fl   created (based on bits and pieces from unidb)
# 2000-09-25 fl   merged tim's splitbin fixes, separate decomposition table
# 2000-09-25 fl   added character type table
# 2000-09-26 fl   added LINEBREAK, DECIMAL, and DIGIT flags/fields (2.0)
# 2000-11-03 fl   expand first/last ranges
# 2001-01-19 fl   added character name tables (2.1)
# 2001-01-21 fl   added decomp compression; dynamic phrasebook threshold
# 2002-09-11 wd   use string methods
# 2002-10-18 mvl  update to Unicode 3.2
# 2002-10-22 mvl  generate NFC tables
# 2002-11-24 mvl  expand all ranges, sort names version-independently
# 2002-11-25 mvl  add UNIDATA_VERSION
# 2004-05-29 perky add east asian width information
# 2006-03-10 mvl  update to Unicode 4.1; add UCD 3.2 delta
# 2008-06-11 gb   add PRINTABLE_MASK for Atsuo Ishimoto's ascii() patch
#
# written by Fredrik Lundh (fredrik@pythonware.com)
#

from os.path import join
import sys

SCRIPT = sys.argv[0]
try:
    DIR = sys.argv[1]
except IndexError:
    DIR = "src"

# The Unicode Database
UNICODE_DATA = join(DIR, "UnicodeData.txt")
UNIHAN = join(DIR, "Unihan.txt")
DERIVED_CORE_PROPERTIES = join(DIR, "DerivedCoreProperties.txt")
LINE_BREAK = join(DIR, "LineBreak.txt")

MANDATORY_LINE_BREAKS = ["BK", "CR", "LF", "NL"]

def parse_hex(s):
    return int(s, 16)

with open(join(DIR, "unicode.c")) as fp:
    for line in [line.strip() for line in fp]:
        if not line.startswith("#define"):
            continue
        _, name, val = line.split()
        if name == "ALPHA_MASK":
            ALPHA_MASK = parse_hex(val)
        elif name == "DECIMAL_MASK":
            DECIMAL_MASK = parse_hex(val)
        elif name == "DIGIT_MASK":
            DIGIT_MASK = parse_hex(val)
        elif name == "NODELTA_MASK":
            NODELTA_MASK = parse_hex(val)
        elif name == "NUMERIC_MASK":
            NUMERIC_MASK = parse_hex(val)

def make_tables():
    print("--- Reading", UNICODE_DATA, "...")
    unicode = UnicodeData(UNICODE_DATA,
                          UNIHAN,
                          DERIVED_CORE_PROPERTIES,
                          LINE_BREAK)
    print(len(list(filter(None, unicode.table))), "characters")
    make_unicodetype(unicode)

def make_cases(filename, chars):
    with open(join(DIR, filename), "w") as fp:
        for codepoint in sorted(chars):
            print("case 0x%04X:" % (codepoint, ), file=fp)

# --------------------------------------------------------------------
# unicode character type tables
def make_unicodetype(unicode):
    # extract unicode types
    table = []
    cache = {}
    index = [0] * len(unicode.chars)
    numeric = {}
    spaces = []
    linebreaks = []

    for char in unicode.chars:
        record = unicode.table[char]
        if record is None:
            continue
        # extract database properties
        category = record.category
        bidirectional = record.bidi
        properties = record.properties
        flags = 0
        delta = True
        if category in ["Lm", "Lt", "Lu", "Ll", "Lo"]:
            flags |= ALPHA_MASK
        if ('Line_Break' in properties) or (bidirectional == "B"):
            linebreaks.append(char)
        if (category == "Zs") or (bidirectional in ("WS", "B", "S")):
            spaces.append(char)
        # use delta predictor for lower if it fits
        lower = parse_hex(record.lowercase) if record.lowercase != "" else char
        lower_d = lower - char
        if -32768 <= lower_d <= 32767:
            # use deltas
            lower = lower_d & 0xffff # To be in range of 16bits
        else:
            flags |= NODELTA_MASK
        # decimal digit, integer digit
        if record.decimal != "":
            flags |= DECIMAL_MASK
        if record.digit != "":
            flags |= DIGIT_MASK
        if record.numeric != "":
            flags |= NUMERIC_MASK
        item = (lower, flags)
        # add entry to index and item tables
        i = cache.get(item)
        if i is None:
            cache[item] = i = len(table)
            table.append(item)
        index[char] = i

    print(len(table), "unique character type entries")
    print(sum(map(len, numeric.values())), "numeric code points")
    print(len(spaces), "whitespace code points")
    print(len(linebreaks), "linebreak code points")

    with open(join(DIR, "entries.inc"), "w") as fp:
        for item in table:
            print("{ %d, %d }," % item, file=fp)

    # split decomposition index table
    index1, index2, shift = splitbins(index)
    with open(join(DIR, "indexes.inc"), "w") as fp:
        print("#define SHIFT", shift, file=fp)
        Array("index1", index1).dump(fp)
        Array("index2", index2).dump(fp)

    make_cases("whitespaces.inc", spaces)
    make_cases("linebreaks.inc", linebreaks)

def iterate_records(filename, sep=";", encoding="utf-8"):
    with open(filename, encoding=encoding) as fp:
        for line in fp:
            data = line.split("#")[0].strip()
            if data == "":
                continue
            yield [s.strip() for s in data.split(sep)]

CHARS_NUM = 0x110000

class Char:
    def __init__(self, name, category, combining, bidi, decimal, digit, numeric, lowercase):
        self.name = name
        self.category = category
        self.combining = combining
        self.bidi = bidi
        self.decimal = decimal
        self.digit = digit
        self.numeric = numeric
        self.lowercase = lowercase
        self.properties = set()

    def __repr__(self):
        attrs = [["name", self.name], ["category", self.category], ["combining", self.combining], ["bidi", self.bidi], ["decimal", self.decimal], ["digit", self.digit], ["numeric", self.numeric], ["lowercase", self.lowercase], ["properties", ", ".join(self.properties)]]
        return "<Char %s>" % (", ".join(["%s=\"%s\"" % (name, val) for name, val in attrs]))

def iterate_codepoint():
    for c in range(0x110000):
        yield c

def extract_range(s):
    if '..' not in s:
        first = parse_hex(s)
        return first, first + 1
    first, last = [parse_hex(c) for c in s.split('..')]
    return first, last + 1

# --------------------------------------------------------------------
# the following support code is taken from the unidb utilities
# Copyright (c) 1999-2000 by Secret Labs AB
# load a unicode-data file from disk
class UnicodeData:
    # Record structure:
    # [ID, name, category, combining, bidi, decomp,  (6)
    #  decimal, digit, numeric, bidi-mirrored, Unicode-1-name, (11)
    #  ISO-comment, uppercase, lowercase, titlecase, ea-width, (16)
    #  derived-props] (17)
    def __init__(self, data, unihan, derivedprops, linebreakprops):
        table = [None] * CHARS_NUM
        for rec in iterate_records(data):
            codepoint = parse_hex(rec[0])
            try:
                c = Char(rec[1], rec[2], rec[3], rec[4], rec[6], rec[7], rec[8], rec[13])
            except IndexError:
                continue
            table[codepoint] = c

        # expand first-last ranges
        field = None
        for c in iterate_codepoint():
            rec = table[c]
            if rec is not None:
                if rec.name.endswith("First>"):
                    rec.name = ""
                    field = rec
                elif rec.name.endswith("Last>"):
                    rec.name = ""
                    field = None
            elif field is not None:
                table[c] = field

        # public attributes
        self.table = table
        self.chars = list(range(CHARS_NUM)) # unicode 3.2

        for r, p in iterate_records(derivedprops):
            if ".." in r:
                first, last = [parse_hex(c) for c in r.split('..')]
                chars = list(range(first, last + 1))
            else:
                chars = [parse_hex(r)]
            for char in chars:
                if table[char] is None:
                    continue
                # Some properties (e.g. Default_Ignorable_Code_Point)
                # apply to unassigned code points; ignore them
                table[char].properties.add(p)

        for rec in iterate_records(linebreakprops):
            if (len(rec) < 2) or (rec[1] not in MANDATORY_LINE_BREAKS):
                continue
            first, last = extract_range(rec[0])
            for char in range(first, last):
                table[char].properties.add("Line_Break")

        for rec in iterate_records(unihan, "\t", "utf-8"):
            if not rec[0].startswith("U+"):
                continue
            try:
                code, tag, value = line.split(None, 3)[:3]
            except ValueError:
                continue
            if tag not in ('kAccountingNumeric', 'kPrimaryNumeric', 'kOtherNumeric'):
                continue
            c = parse_hex(code[len("U+"):])
            # Patch the numeric field
            if table[c] is None:
                continue
            table[c].numeric = value.strip().replace(',', '')

# stuff to deal with arrays of unsigned integers
class Array:
    def __init__(self, name, data):
        self.name = name
        self.data = data

    def dump(self, file):
        # write data to file, as a C array
        size = get_size(self.data)
        print(self.name + ":", size * len(self.data), "bytes", file=sys.stderr)
        file.write("static ")
        if size == 1:
            file.write("unsigned char")
        elif size == 2:
            file.write("unsigned short")
        else:
            file.write("unsigned int")
        file.write(" " + self.name + "[] = {\n")
        if self.data:
            s = "    "
            for item in self.data:
                i = str(item) + ", "
                if len(s) + len(i) > 78:
                    file.write(s + "\n")
                    s = "    " + i
                else:
                    s = s + i
            if s.strip():
                file.write(s + "\n")
        file.write("};\n\n")

def get_size(data):
    # return smallest possible integer size for the given array
    maxdata = max(data)
    if maxdata < 256:
        return 1
    if maxdata < 65536:
        return 2
    return 4

def splitbins(t):
    """t -> (t1, t2, shift).  Split a table to save space.

    t is a sequence of ints.  This function can be useful to save space if
    many of the ints are the same.  t1 and t2 are lists of ints, and shift
    is an int, chosen to minimize the combined size of t1 and t2 (in C
    code), and where for each i in range(len(t)),
        t[i] == t2[(t1[i >> shift] << shift) + (i & mask)]
    where mask is a bitmask isolating the last "shift" bits.
    """

    def dump(t1, t2, shift, bytes):
        print("%d + %d bins at shift %d; %d bytes" % (len(t1), len(t2), shift, bytes), file=sys.stderr)
    print("Size of original table:", len(t) * get_size(t), "bytes", file=sys.stderr)
    n = len(t)-1    # last valid index
    maxshift = 0    # the most we can shift n and still have something left
    if n > 0:
        while n >> 1:
            n >>= 1
            maxshift += 1
    del n
    bytes = sys.maxsize  # smallest total size so far
    t = tuple(t)    # so slices can be dict keys
    for shift in range(maxshift + 1):
        t1 = []
        t2 = []
        size = 2**shift
        bincache = {}
        for i in range(0, len(t), size):
            bin = t[i:i+size]
            index = bincache.get(bin)
            if index is None:
                index = len(t2)
                bincache[bin] = index
                t2.extend(bin)
            t1.append(index >> shift)
        # determine memory size
        b = len(t1) * get_size(t1) + len(t2) * get_size(t2)
        dump(t1, t2, shift, b)
        if b < bytes:
            best = t1, t2, shift
            bytes = b
    t1, t2, shift = best
    print("Best:", end=' ', file=sys.stderr)
    dump(t1, t2, shift, bytes)
    if __debug__:
        # exhaustively verify that the decomposition is correct
        mask = ~((~0) << shift) # i.e., low-bit mask of shift bits
        for i in range(len(t)):
            assert t[i] == t2[(t1[i >> shift] << shift) + (i & mask)]
    return best

if __name__ == "__main__":
    make_tables()

# vim: tabstop=4 shiftwidth=4 expandtab softtabstop=4
<commit_msg>tools/makeunicodedata.py: Improved to accept both of a source directory and a destination directory.<commit_after>#
# (re)generate unicode property and type databases
#
# this script converts a unicode 3.2 database file to
# Modules/unicodedata_db.h, Modules/unicodename_db.h,
# and Objects/unicodetype_db.h
#
# history:
# 2000-09-24 fl   created (based on bits and pieces from unidb)
# 2000-09-25 fl   merged tim's splitbin fixes, separate decomposition table
# 2000-09-25 fl   added character type table
# 2000-09-26 fl   added LINEBREAK, DECIMAL, and DIGIT flags/fields (2.0)
# 2000-11-03 fl   expand first/last ranges
# 2001-01-19 fl   added character name tables (2.1)
# 2001-01-21 fl   added decomp compression; dynamic phrasebook threshold
# 2002-09-11 wd   use string methods
# 2002-10-18 mvl  update to Unicode 3.2
# 2002-10-22 mvl  generate NFC tables
# 2002-11-24 mvl  expand all ranges, sort names version-independently
# 2002-11-25 mvl  add UNIDATA_VERSION
# 2004-05-29 perky add east asian width information
# 2006-03-10 mvl  update to Unicode 4.1; add UCD 3.2 delta
# 2008-06-11 gb   add PRINTABLE_MASK for Atsuo Ishimoto's ascii() patch
#
# written by Fredrik Lundh (fredrik@pythonware.com)
#

from os.path import join
import sys

SCRIPT = sys.argv[0]
try:
    src_dir = sys.argv[1]
except IndexError:
    src_dir = "src"
try:
    dest_dir = sys.argv[2]
except IndexError:
    dest_dir = src_dir

# The Unicode Database
UNICODE_DATA = join(src_dir, "UnicodeData.txt")
UNIHAN = join(src_dir, "Unihan.txt")
DERIVED_CORE_PROPERTIES = join(src_dir, "DerivedCoreProperties.txt")
LINE_BREAK = join(src_dir, "LineBreak.txt")

MANDATORY_LINE_BREAKS = ["BK", "CR", "LF", "NL"]

def parse_hex(s):
    return int(s, 16)

with open(join(src_dir, "unicode.c")) as fp:
    for line in [line.strip() for line in fp]:
        if not line.startswith("#define"):
            continue
        _, name, val = line.split()
        if name == "ALPHA_MASK":
            ALPHA_MASK = parse_hex(val)
        elif name == "DECIMAL_MASK":
            DECIMAL_MASK = parse_hex(val)
        elif name == "DIGIT_MASK":
            DIGIT_MASK = parse_hex(val)
        elif name == "NODELTA_MASK":
            NODELTA_MASK = parse_hex(val)
        elif name == "NUMERIC_MASK":
            NUMERIC_MASK = parse_hex(val)

def make_tables():
    print("--- Reading", UNICODE_DATA, "...")
    unicode = UnicodeData(UNICODE_DATA,
                          UNIHAN,
                          DERIVED_CORE_PROPERTIES,
                          LINE_BREAK)
    print(len(list(filter(None, unicode.table))), "characters")
    make_unicodetype(unicode)

def make_cases(filename, chars):
    with open(join(dest_dir, filename), "w") as fp:
        for codepoint in sorted(chars):
            print("case 0x%04X:" % (codepoint, ), file=fp)

# --------------------------------------------------------------------
# unicode character type tables
def make_unicodetype(unicode):
    # extract unicode types
    table = []
    cache = {}
    index = [0] * len(unicode.chars)
    numeric = {}
    spaces = []
    linebreaks = []

    for char in unicode.chars:
        record = unicode.table[char]
        if record is None:
            continue
        # extract database properties
        category = record.category
        bidirectional = record.bidi
        properties = record.properties
        flags = 0
        delta = True
        if category in ["Lm", "Lt", "Lu", "Ll", "Lo"]:
            flags |= ALPHA_MASK
        if ('Line_Break' in properties) or (bidirectional == "B"):
            linebreaks.append(char)
        if (category == "Zs") or (bidirectional in ("WS", "B", "S")):
            spaces.append(char)
        # use delta predictor for lower if it fits
        lower = parse_hex(record.lowercase) if record.lowercase != "" else char
        lower_d = lower - char
        if -32768 <= lower_d <= 32767:
            # use deltas
            lower = lower_d & 0xffff # To be in range of 16bits
        else:
            flags |= NODELTA_MASK
        # decimal digit, integer digit
        if record.decimal != "":
            flags |= DECIMAL_MASK
        if record.digit != "":
            flags |= DIGIT_MASK
        if record.numeric != "":
            flags |= NUMERIC_MASK
        item = (lower, flags)
        # add entry to index and item tables
        i = cache.get(item)
        if i is None:
            cache[item] = i = len(table)
            table.append(item)
        index[char] = i

    print(len(table), "unique character type entries")
    print(sum(map(len, numeric.values())), "numeric code points")
    print(len(spaces), "whitespace code points")
    print(len(linebreaks), "linebreak code points")

    with open(join(dest_dir, "entries.inc"), "w") as fp:
        for item in table:
            print("{ %d, %d }," % item, file=fp)

    # split decomposition index table
    index1, index2, shift = splitbins(index)
    with open(join(dest_dir, "indexes.inc"), "w") as fp:
        print("#define SHIFT", shift, file=fp)
        Array("index1", index1).dump(fp)
        Array("index2", index2).dump(fp)

    make_cases("whitespaces.inc", spaces)
    make_cases("linebreaks.inc", linebreaks)

def iterate_records(filename, sep=";", encoding="utf-8"):
    with open(filename, encoding=encoding) as fp:
        for line in fp:
            data = line.split("#")[0].strip()
            if data == "":
                continue
            yield [s.strip() for s in data.split(sep)]

CHARS_NUM = 0x110000

class Char:
    def __init__(self, name, category, combining, bidi, decimal, digit, numeric, lowercase):
        self.name = name
        self.category = category
        self.combining = combining
        self.bidi = bidi
        self.decimal = decimal
        self.digit = digit
        self.numeric = numeric
        self.lowercase = lowercase
        self.properties = set()

    def __repr__(self):
        attrs = [["name", self.name], ["category", self.category], ["combining", self.combining], ["bidi", self.bidi], ["decimal", self.decimal], ["digit", self.digit], ["numeric", self.numeric], ["lowercase", self.lowercase], ["properties", ", ".join(self.properties)]]
        return "<Char %s>" % (", ".join(["%s=\"%s\"" % (name, val) for name, val in attrs]))

def iterate_codepoint():
    for c in range(0x110000):
        yield c

def extract_range(s):
    if '..' not in s:
        first = parse_hex(s)
        return first, first + 1
    first, last = [parse_hex(c) for c in s.split('..')]
    return first, last + 1

# --------------------------------------------------------------------
# the following support code is taken from the unidb utilities
# Copyright (c) 1999-2000 by Secret Labs AB
# load a unicode-data file from disk
class UnicodeData:
    # Record structure:
    # [ID, name, category, combining, bidi, decomp,  (6)
    #  decimal, digit, numeric, bidi-mirrored, Unicode-1-name, (11)
    #  ISO-comment, uppercase, lowercase, titlecase, ea-width, (16)
    #  derived-props] (17)
    def __init__(self, data, unihan, derivedprops, linebreakprops):
        table = [None] * CHARS_NUM
        for rec in iterate_records(data):
            codepoint = parse_hex(rec[0])
            try:
                c = Char(rec[1], rec[2], rec[3], rec[4], rec[6], rec[7], rec[8], rec[13])
            except IndexError:
                continue
            table[codepoint] = c

        # expand first-last ranges
        field = None
        for c in iterate_codepoint():
            rec = table[c]
            if rec is not None:
                if rec.name.endswith("First>"):
                    rec.name = ""
                    field = rec
                elif rec.name.endswith("Last>"):
                    rec.name = ""
                    field = None
            elif field is not None:
                table[c] = field

        # public attributes
        self.table = table
        self.chars = list(range(CHARS_NUM)) # unicode 3.2

        for r, p in iterate_records(derivedprops):
            if ".." in r:
                first, last = [parse_hex(c) for c in r.split('..')]
                chars = list(range(first, last + 1))
            else:
                chars = [parse_hex(r)]
            for char in chars:
                if table[char] is None:
                    continue
                # Some properties (e.g. Default_Ignorable_Code_Point)
                # apply to unassigned code points; ignore them
                table[char].properties.add(p)

        for rec in iterate_records(linebreakprops):
            if (len(rec) < 2) or (rec[1] not in MANDATORY_LINE_BREAKS):
                continue
            first, last = extract_range(rec[0])
            for char in range(first, last):
                table[char].properties.add("Line_Break")

        for rec in iterate_records(unihan, "\t", "utf-8"):
            if not rec[0].startswith("U+"):
                continue
            try:
                code, tag, value = line.split(None, 3)[:3]
            except ValueError:
                continue
            if tag not in ('kAccountingNumeric', 'kPrimaryNumeric', 'kOtherNumeric'):
                continue
            c = parse_hex(code[len("U+"):])
            # Patch the numeric field
            if table[c] is None:
                continue
            table[c].numeric = value.strip().replace(',', '')

# stuff to deal with arrays of unsigned integers
class Array:
    def __init__(self, name, data):
        self.name = name
        self.data = data

    def dump(self, file):
        # write data to file, as a C array
        size = get_size(self.data)
        print(self.name + ":", size * len(self.data), "bytes", file=sys.stderr)
        file.write("static ")
        if size == 1:
            file.write("unsigned char")
        elif size == 2:
            file.write("unsigned short")
        else:
            file.write("unsigned int")
        file.write(" " + self.name + "[] = {\n")
        if self.data:
            s = "    "
            for item in self.data:
                i = str(item) + ", "
                if len(s) + len(i) > 78:
                    file.write(s + "\n")
                    s = "    " + i
                else:
                    s = s + i
            if s.strip():
                file.write(s + "\n")
        file.write("};\n\n")

def get_size(data):
    # return smallest possible integer size for the given array
    maxdata = max(data)
    if maxdata < 256:
        return 1
    if maxdata < 65536:
        return 2
    return 4

def splitbins(t):
    """t -> (t1, t2, shift).  Split a table to save space.

    t is a sequence of ints.  This function can be useful to save space if
    many of the ints are the same.  t1 and t2 are lists of ints, and shift
    is an int, chosen to minimize the combined size of t1 and t2 (in C
    code), and where for each i in range(len(t)),
        t[i] == t2[(t1[i >> shift] << shift) + (i & mask)]
    where mask is a bitmask isolating the last "shift" bits.
    """

    def dump(t1, t2, shift, bytes):
        print("%d + %d bins at shift %d; %d bytes" % (len(t1), len(t2), shift, bytes), file=sys.stderr)
    print("Size of original table:", len(t) * get_size(t), "bytes", file=sys.stderr)
    n = len(t)-1    # last valid index
    maxshift = 0    # the most we can shift n and still have something left
    if n > 0:
        while n >> 1:
            n >>= 1
            maxshift += 1
    del n
    bytes = sys.maxsize  # smallest total size so far
    t = tuple(t)    # so slices can be dict keys
    for shift in range(maxshift + 1):
        t1 = []
        t2 = []
        size = 2**shift
        bincache = {}
        for i in range(0, len(t), size):
            bin = t[i:i+size]
            index = bincache.get(bin)
            if index is None:
                index = len(t2)
                bincache[bin] = index
                t2.extend(bin)
            t1.append(index >> shift)
        # determine memory size
        b = len(t1) * get_size(t1) + len(t2) * get_size(t2)
        dump(t1, t2, shift, b)
        if b < bytes:
            best = t1, t2, shift
            bytes = b
    t1, t2, shift = best
    print("Best:", end=' ', file=sys.stderr)
    dump(t1, t2, shift, bytes)
    if __debug__:
        # exhaustively verify that the decomposition is correct
        mask = ~((~0) << shift) # i.e., low-bit mask of shift bits
        for i in range(len(t)):
            assert t[i] == t2[(t1[i >> shift] << shift) + (i & mask)]
    return best

if __name__ == "__main__":
    make_tables()

# vim: tabstop=4 shiftwidth=4 expandtab softtabstop=4
<|endoftext|>


<commit_before>"""
.. module: lemur.plugins.lemur_aws.s3
    :platform: Unix
    :synopsis: Contains helper functions for interactive with AWS S3 Apis.
    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
    :license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from botocore.exceptions import ClientError
from flask import current_app
from lemur.extensions import sentry

from .sts import sts_client

@sts_client("s3", service_type="resource")
def put(bucket_name, region_name, prefix, data, encrypt, **kwargs):
    """
    Use STS to write to an S3 bucket
    """
    bucket = kwargs["resource"].Bucket(bucket_name)
    current_app.logger.debug(
        "Persisting data to S3. Bucket: {0} Prefix: {1}".format(bucket_name, prefix)
    )

    # get data ready for writing
    if isinstance(data, str):
        data = data.encode("utf-8")

    if encrypt:
        bucket.put_object(
            Key=prefix,
            Body=data,
            ACL="bucket-owner-full-control",
            ServerSideEncryption="AES256",
        )
    else:
        try:
            bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control")
        except ClientError:
            sentry.captureException()

@sts_client("s3", service_type="client")
def delete(bucket_name, prefix, **kwargs):
    """
    Use STS to delete an object
    """
    try:
        response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefix)
        current_app.logger.debug(f"Delete data from S3."
                                 f"Bucket: {bucket_name},"
                                 f"Prefix: {prefix},"
                                 f"Status_code: {response}")
        return response['ResponseMetadata']['HTTPStatusCode'] < 300
    except ClientError:
        sentry.captureException()

@sts_client("s3", service_type="client")
def get(bucket_name, prefix, **kwargs):
    """
    Use STS to get an object
    """
    try:
        response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefix)
        current_app.logger.debug(f"Get data from S3. Bucket: {bucket_name},"
                                 f"Prefix: {prefix}")
        return response['Body'].read().decode("utf-8")
    except ClientError:
        sentry.captureException()
<commit_msg>better return arguments<commit_after>"""
.. module: lemur.plugins.lemur_aws.s3
    :platform: Unix
    :synopsis: Contains helper functions for interactive with AWS S3 Apis.
    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
    :license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from botocore.exceptions import ClientError
from flask import current_app
from lemur.extensions import sentry

from .sts import sts_client

@sts_client("s3", service_type="resource")
def put(bucket_name, region_name, prefix, data, encrypt, **kwargs):
    """
    Use STS to write to an S3 bucket
    """
    bucket = kwargs["resource"].Bucket(bucket_name)
    current_app.logger.debug(
        "Persisting data to S3. Bucket: {0} Prefix: {1}".format(bucket_name, prefix)
    )

    # get data ready for writing
    if isinstance(data, str):
        data = data.encode("utf-8")

    if encrypt:
        bucket.put_object(
            Key=prefix,
            Body=data,
            ACL="bucket-owner-full-control",
            ServerSideEncryption="AES256",
        )
    else:
        try:
            bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control")
            return True
        except ClientError:
            sentry.captureException()
            return False

@sts_client("s3", service_type="client")
def delete(bucket_name, prefix, **kwargs):
    """
    Use STS to delete an object
    """
    try:
        response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefix)
        current_app.logger.debug(f"Delete data from S3."
                                 f"Bucket: {bucket_name},"
                                 f"Prefix: {prefix},"
                                 f"Status_code: {response}")
        return response['ResponseMetadata']['HTTPStatusCode'] < 300
    except ClientError:
        sentry.captureException()
        return False

@sts_client("s3", service_type="client")
def get(bucket_name, prefix, **kwargs):
    """
    Use STS to get an object
    """
    try:
        response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefix)
        current_app.logger.debug(f"Get data from S3. Bucket: {bucket_name},"
                                 f"Prefix: {prefix}")
        return response['Body'].read().decode("utf-8")
    except ClientError:
        sentry.captureException()
        return None
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""A file to just echo your name."""

print 'Hello Snoopy!'
<commit_msg>Added task_02.py<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""A file to just echo your name."""

print 'Hello Kristell Vasquez!'
<|endoftext|>


<commit_before>#! usr/bin/env python
# *-* coding: utf-8 *-*
"""Docstring"""

import decimal

def lexicographics(to_analyze):
    """This is a docstring.

    Arg:
        to_analyze(str): a required string
    Return:
        To return values as tuples, in the order defined
    Example:
        >>> import task_03
        >>> task_03.lexicographics('''Don't stop believing,
        Hold on to that feeling.''')
        (5, 3, Decimal(4.0))

        >>> import task_03
        >>> import data
        >>> task_03.lexicographics(data.SHAKESPEARE)
        (12, 5, Decimal('8.14'))
    """
    split_one = to_analyze.split('\n')
    length = []

    for words in split_one:
        split_two = words.split()
        length.append(len(split_two))
        t_length = len(split_one)
        overall_length = ((max(length), min(length)),
                          (sum(length) / decimal.Decimal(t_length)))
    return overall_length
<commit_msg>removed extra tuple in output<commit_after>#! usr/bin/env python
# *-* coding: utf-8 *-*
"""Docstring"""

import decimal

def lexicographics(to_analyze):
    """This is a docstring.

    Arg:
        to_analyze(str): a required string
    Return:
        To return values as tuples, in the order defined
    Example:
        >>> import task_03
        >>> task_03.lexicographics('''Don't stop believing,
        Hold on to that feeling.''')
        (5, 3, Decimal(4.0))

        >>> import task_03
        >>> import data
        >>> task_03.lexicographics(data.SHAKESPEARE)
        (12, 5, Decimal('8.14'))
    """
    split_one = to_analyze.split('\n')
    length = []

    for words in split_one:
        split_two = words.split()
        length.append(len(split_two))
        t_length = len(split_one)
        overall_length = (max(length), min(length),
                          sum(length) / decimal.Decimal(t_length))

    return overall_length
<|endoftext|>