<commit_before>"""General utility functions."""

import datetime as dt
import logging as lg
import os
import sys
import unicodedata

from . import settings

def citation():
    """
    Print the OSMnx package's citation information.

    Boeing, G. 2017. OSMnx: New Methods for Acquiring, Constructing, Analyzing,
    and Visualizing Complex Street Networks. Computers, Environment and Urban
    Systems, 65(126-139). https://doi.org/10.1016/j.compenvurbsys.2017.05.004

    Returns
    -------
    None
    """
    cite = (
        "Citation:\n\n"
        "Boeing, G. 2017. OSMnx: New Methods for Acquiring, "
        "Constructing, Analyzing, and Visualizing Complex Street "
        "Networks. Computers, Environment and Urban Systems, 65(126-139). "
        "https://doi.org/10.1016/j.compenvurbsys.2017.05.004\n\n"
        "BibTeX entry for LaTeX users:\n\n"
        "@article{boeing_osmnx_2017,\n"
        "    title = {{OSMnx}: {New} {Methods} for {Acquiring}, "
        "{Constructing}, {Analyzing}, and {Visualizing} {Complex} "
        "{Street} {Networks}},\n"
        "    volume = {65},\n"
        "    doi = {10.1016/j.compenvurbsys.2017.05.004},\n"
        "    number = {126-139},\n"
        "    journal = {Computers, Environment and Urban Systems},\n"
        "    author = {Boeing, Geoff},\n"
        "    year = {2017}\n"
        "}"
    )

    print(cite)

def ts(style="datetime", template=None):
    """
    Get current timestamp as string.

    Parameters
    ----------
    style : string
        format the timestamp with this built-in template. must be one
        of {'datetime', 'date', 'time'}
    template : string
        if not None, format the timestamp with this template instead of
        one of the built-in styles

    Returns
    -------
    ts : string
        the string timestamp
    """
    if template is None:
        if style == "datetime":
            template = "{:%Y-%m-%d %H:%M:%S}"
        elif style == "date":
            template = "{:%Y-%m-%d}"
        elif style == "time":
            template = "{:%H:%M:%S}"
        else:
            raise ValueError(f'unrecognized timestamp style "{style}"')

    ts = template.format(dt.datetime.now())
    return ts

def config(
    data_folder=settings.data_folder,
    logs_folder=settings.logs_folder,
    imgs_folder=settings.imgs_folder,
    cache_folder=settings.cache_folder,
    use_cache=settings.use_cache,
    log_file=settings.log_file,
    log_console=settings.log_console,
    log_level=settings.log_level,
    log_name=settings.log_name,
    log_filename=settings.log_filename,
    useful_tags_node=settings.useful_tags_node,
    useful_tags_way=settings.useful_tags_way,
    osm_xml_node_attrs=settings.osm_xml_node_attrs,
    osm_xml_node_tags=settings.osm_xml_node_tags,
    osm_xml_way_attrs=settings.osm_xml_way_attrs,
    osm_xml_way_tags=settings.osm_xml_way_tags,
    overpass_settings=settings.overpass_settings,
    timeout=settings.timeout,
    memory=settings.memory,
    max_query_area_size=settings.max_query_area_size,
    default_access=settings.default_access,
    default_crs=settings.default_crs,
    default_user_agent=settings.default_user_agent,
    default_referer=settings.default_referer,
    default_accept_language=settings.default_accept_language,
    nominatim_endpoint=settings.nominatim_endpoint,
    nominatim_key=settings.nominatim_key,
    overpass_endpoint=settings.overpass_endpoint,
    all_oneway=settings.all_oneway,
    elevation_provider=settings.elevation_provider,
):
    """
    Configure OSMnx by setting the default global settings' values.

    Any parameters not passed by the caller are set to their original default
    values.

    Parameters
    ----------
    data_folder : string
        path to folder in which to save/load data files by default
    logs_folder : string
        path to folder in which to save log files
    imgs_folder : string
        path to folder in which to save plotted figures by default
    cache_folder : string
        path to folder in which to save/load HTTP response cache
    use_cache : bool
        if True, cache HTTP responses locally instead of calling API
        repetitively for the same request
    log_file : bool
        if True, save log output to a file in logs_folder
    log_console : bool
        if True, print log output to the console (terminal window)
    log_level : int
        one of the logger.level constants
    log_name : string
        name of the logger
    log_filename : string
        name of the log file
    useful_tags_node : list
        OSM "node" tags to add as graph node attributes, when present
    useful_tags_way : list
        OSM "way" tags to add as graph edge attributes, when present
    osm_xml_node_attrs : list
        node attributes for saving .osm XML files with save_graph_xml function
    osm_xml_node_tags : list
        node tags for saving .osm XML files with save_graph_xml function
    osm_xml_way_attrs : list
        edge attributes for saving .osm XML files with save_graph_xml function
    osm_xml_way_tags : list
        edge tags for for saving .osm XML files with save_graph_xml function
    overpass_settings : string
        Settings string for overpass queries. For example, to query historical
        OSM data as of a certain date:
        `'[out:json][timeout:90][date:"2019-10-28T19:20:00Z"]'`.
        Use with caution.
    timeout : int
        the timeout interval for the HTTP request and for API to use while
        running the query
    memory : int
        Overpass server memory allocation size for the query, in bytes. If
        None, server will use its default allocation size. Use with caution.
    max_query_area_size : int
        maximum area for any part of the geometry in meters: any polygon
        bigger than this will get divided up for multiple queries to API
        (default 50km x 50km)
    default_access : string
        default filter for OSM "access" key
    default_crs : string
        default coordinate reference system to set when creating graphs
    default_user_agent : string
        HTTP header user-agent
    default_referer : string
        HTTP header referer
    default_accept_language : string
        HTTP header accept-language
    nominatim_endpoint : string
        the API endpoint to use for nominatim queries
    nominatim_key : string
        your API key, if you are using an endpoint that requires one
    overpass_endpoint : string
        the API endpoint to use for overpass queries
    all_oneway : bool
        if True, forces all ways to be loaded as oneway ways, preserving
        the original order of nodes stored in the OSM way XML. Only use if
        specifically saving to .osm XML file with save_graph_xml function.
    elevation_provider : string
        the API provider to use for adding node elevations, can be either
        "google" or "airmap"

    Returns
    -------
    None
    """
    # set each global setting to the passed-in value
    settings.use_cache = use_cache
    settings.cache_folder = cache_folder
    settings.data_folder = data_folder
    settings.imgs_folder = imgs_folder
    settings.logs_folder = logs_folder
    settings.log_console = log_console
    settings.log_file = log_file
    settings.log_level = log_level
    settings.log_name = log_name
    settings.log_filename = log_filename
    settings.useful_tags_node = useful_tags_node
    settings.useful_tags_way = useful_tags_way
    settings.osm_xml_node_attrs = osm_xml_node_attrs
    settings.osm_xml_node_tags = osm_xml_node_tags
    settings.osm_xml_way_attrs = osm_xml_way_attrs
    settings.osm_xml_way_tags = osm_xml_way_tags
    settings.overpass_settings = overpass_settings
    settings.timeout = timeout
    settings.memory = memory
    settings.max_query_area_size = max_query_area_size
    settings.default_access = default_access
    settings.default_crs = default_crs
    settings.default_user_agent = default_user_agent
    settings.default_referer = default_referer
    settings.default_accept_language = default_accept_language
    settings.nominatim_endpoint = nominatim_endpoint
    settings.nominatim_key = nominatim_key
    settings.overpass_endpoint = overpass_endpoint
    settings.all_oneway = all_oneway
    settings.elevation_provider = elevation_provider

    # if logging is turned on, log that we are configured
    if settings.log_file or settings.log_console:
        log("Configured osmnx")

def log(message, level=None, name=None, filename=None):
    """
    Write a message to the logger.

    This logs to file and/or prints to the console (terminal), depending on
    the current configuration of settings.log_file and settings.log_console.

    Parameters
    ----------
    message : string
        the message to log
    level : int
        one of the logger.level constants
    name : string
        name of the logger
    filename : string
        name of the log file

    Returns
    -------
    None
    """
    if level is None:
        level = settings.log_level
    if name is None:
        name = settings.log_name
    if filename is None:
        filename = settings.log_filename

    # if logging to file is turned on
    if settings.log_file:
        # get the current logger (or create a new one, if none), then log
        # message at requested level
        logger = _get_logger(level=level, name=name, filename=filename)
        if level == lg.DEBUG:
            logger.debug(message)
        elif level == lg.INFO:
            logger.info(message)
        elif level == lg.WARNING:
            logger.warning(message)
        elif level == lg.ERROR:
            logger.error(message)

    # if logging to console, convert message to ascii and print to console
    if settings.log_console:
        # capture current stdout, then switch it to the console, print the
        # message, then switch back to what had been the stdout. this prevents
        # logging to notebook in jupyter: instead, it goes to terminal
        standard_out = sys.stdout
        sys.stdout = sys.__stdout__

        # prepend timestamp
        message = f"{ts()} {message}"

        # convert to ascii so it doesn't break windows terminals
        message = (
            unicodedata.normalize("NFKD", str(message)).encode("ascii", errors="replace").decode()
        )
        print(message)
        sys.stdout = standard_out

def _get_logger(level=None, name=None, filename=None):
    """
    Create a logger or return the current one if already instantiated.

    Parameters
    ----------
    level : int
        one of the logger.level constants
    name : string
        name of the logger
    filename : string
        name of the log file

    Returns
    -------
    logger : logging.logger
    """
    if level is None:
        level = settings.log_level
    if name is None:
        name = settings.log_name
    if filename is None:
        filename = settings.log_filename

    logger = lg.getLogger(name)

    # if a logger with this name is not already set up
    if not getattr(logger, "handler_set", None):

        # get today's date and construct a log filename
        log_filename = os.path.join(settings.logs_folder, f'{filename}_{ts(style="date")}.log')

        # if the logs folder does not already exist, create it
        if not os.path.exists(settings.logs_folder):
            os.makedirs(settings.logs_folder)

        # create file handler and log formatter and set them up
        handler = lg.FileHandler(log_filename, encoding="utf-8")
        formatter = lg.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s")
        handler.setFormatter(formatter)
        logger.addHandler(handler)
        logger.setLevel(level)
        logger.handler_set = True

    return logger
<commit_msg>expose bidirectional_network_types setting in ox.config<commit_after>"""General utility functions."""

import datetime as dt
import logging as lg
import os
import sys
import unicodedata

from . import settings

def citation():
    """
    Print the OSMnx package's citation information.

    Boeing, G. 2017. OSMnx: New Methods for Acquiring, Constructing, Analyzing,
    and Visualizing Complex Street Networks. Computers, Environment and Urban
    Systems, 65(126-139). https://doi.org/10.1016/j.compenvurbsys.2017.05.004

    Returns
    -------
    None
    """
    cite = (
        "Citation:\n\n"
        "Boeing, G. 2017. OSMnx: New Methods for Acquiring, "
        "Constructing, Analyzing, and Visualizing Complex Street "
        "Networks. Computers, Environment and Urban Systems, 65(126-139). "
        "https://doi.org/10.1016/j.compenvurbsys.2017.05.004\n\n"
        "BibTeX entry for LaTeX users:\n\n"
        "@article{boeing_osmnx_2017,\n"
        "    title = {{OSMnx}: {New} {Methods} for {Acquiring}, "
        "{Constructing}, {Analyzing}, and {Visualizing} {Complex} "
        "{Street} {Networks}},\n"
        "    volume = {65},\n"
        "    doi = {10.1016/j.compenvurbsys.2017.05.004},\n"
        "    number = {126-139},\n"
        "    journal = {Computers, Environment and Urban Systems},\n"
        "    author = {Boeing, Geoff},\n"
        "    year = {2017}\n"
        "}"
    )

    print(cite)

def ts(style="datetime", template=None):
    """
    Get current timestamp as string.

    Parameters
    ----------
    style : string
        format the timestamp with this built-in template. must be one
        of {'datetime', 'date', 'time'}
    template : string
        if not None, format the timestamp with this template instead of
        one of the built-in styles

    Returns
    -------
    ts : string
        the string timestamp
    """
    if template is None:
        if style == "datetime":
            template = "{:%Y-%m-%d %H:%M:%S}"
        elif style == "date":
            template = "{:%Y-%m-%d}"
        elif style == "time":
            template = "{:%H:%M:%S}"
        else:
            raise ValueError(f'unrecognized timestamp style "{style}"')

    ts = template.format(dt.datetime.now())
    return ts

def config(
    data_folder=settings.data_folder,
    logs_folder=settings.logs_folder,
    imgs_folder=settings.imgs_folder,
    cache_folder=settings.cache_folder,
    use_cache=settings.use_cache,
    log_file=settings.log_file,
    log_console=settings.log_console,
    log_level=settings.log_level,
    log_name=settings.log_name,
    log_filename=settings.log_filename,
    useful_tags_node=settings.useful_tags_node,
    useful_tags_way=settings.useful_tags_way,
    bidirectional_network_types=settings.bidirectional_network_types,
    osm_xml_node_attrs=settings.osm_xml_node_attrs,
    osm_xml_node_tags=settings.osm_xml_node_tags,
    osm_xml_way_attrs=settings.osm_xml_way_attrs,
    osm_xml_way_tags=settings.osm_xml_way_tags,
    overpass_settings=settings.overpass_settings,
    timeout=settings.timeout,
    memory=settings.memory,
    max_query_area_size=settings.max_query_area_size,
    default_access=settings.default_access,
    default_crs=settings.default_crs,
    default_user_agent=settings.default_user_agent,
    default_referer=settings.default_referer,
    default_accept_language=settings.default_accept_language,
    nominatim_endpoint=settings.nominatim_endpoint,
    nominatim_key=settings.nominatim_key,
    overpass_endpoint=settings.overpass_endpoint,
    all_oneway=settings.all_oneway,
    elevation_provider=settings.elevation_provider,
):
    """
    Configure OSMnx by setting the default global settings' values.

    Any parameters not passed by the caller are set to their original default
    values.

    Parameters
    ----------
    data_folder : string
        path to folder in which to save/load data files by default
    logs_folder : string
        path to folder in which to save log files
    imgs_folder : string
        path to folder in which to save plotted figures by default
    cache_folder : string
        path to folder in which to save/load HTTP response cache
    use_cache : bool
        if True, cache HTTP responses locally instead of calling API
        repetitively for the same request
    log_file : bool
        if True, save log output to a file in logs_folder
    log_console : bool
        if True, print log output to the console (terminal window)
    log_level : int
        one of the logger.level constants
    log_name : string
        name of the logger
    log_filename : string
        name of the log file
    useful_tags_node : list
        OSM "node" tags to add as graph node attributes, when present
    useful_tags_way : list
        OSM "way" tags to add as graph edge attributes, when present
    bidirectional_network_types : list
        network types for which a fully bidirectional graph will be created
    osm_xml_node_attrs : list
        node attributes for saving .osm XML files with save_graph_xml function
    osm_xml_node_tags : list
        node tags for saving .osm XML files with save_graph_xml function
    osm_xml_way_attrs : list
        edge attributes for saving .osm XML files with save_graph_xml function
    osm_xml_way_tags : list
        edge tags for for saving .osm XML files with save_graph_xml function
    overpass_settings : string
        Settings string for overpass queries. For example, to query historical
        OSM data as of a certain date:
        `'[out:json][timeout:90][date:"2019-10-28T19:20:00Z"]'`.
        Use with caution.
    timeout : int
        the timeout interval for the HTTP request and for API to use while
        running the query
    memory : int
        Overpass server memory allocation size for the query, in bytes. If
        None, server will use its default allocation size. Use with caution.
    max_query_area_size : int
        maximum area for any part of the geometry in meters: any polygon
        bigger than this will get divided up for multiple queries to API
        (default 50km x 50km)
    default_access : string
        default filter for OSM "access" key
    default_crs : string
        default coordinate reference system to set when creating graphs
    default_user_agent : string
        HTTP header user-agent
    default_referer : string
        HTTP header referer
    default_accept_language : string
        HTTP header accept-language
    nominatim_endpoint : string
        the API endpoint to use for nominatim queries
    nominatim_key : string
        your API key, if you are using an endpoint that requires one
    overpass_endpoint : string
        the API endpoint to use for overpass queries
    all_oneway : bool
        if True, forces all ways to be loaded as oneway ways, preserving
        the original order of nodes stored in the OSM way XML. Only use if
        specifically saving to .osm XML file with save_graph_xml function.
    elevation_provider : string
        the API provider to use for adding node elevations, can be either
        "google" or "airmap"

    Returns
    -------
    None
    """
    # set each global setting to the passed-in value
    settings.use_cache = use_cache
    settings.cache_folder = cache_folder
    settings.data_folder = data_folder
    settings.imgs_folder = imgs_folder
    settings.logs_folder = logs_folder
    settings.log_console = log_console
    settings.log_file = log_file
    settings.log_level = log_level
    settings.log_name = log_name
    settings.log_filename = log_filename
    settings.useful_tags_node = useful_tags_node
    settings.useful_tags_way = useful_tags_way
    settings.osm_xml_node_attrs = osm_xml_node_attrs
    settings.osm_xml_node_tags = osm_xml_node_tags
    settings.osm_xml_way_attrs = osm_xml_way_attrs
    settings.osm_xml_way_tags = osm_xml_way_tags
    settings.overpass_settings = overpass_settings
    settings.timeout = timeout
    settings.memory = memory
    settings.max_query_area_size = max_query_area_size
    settings.default_access = default_access
    settings.default_crs = default_crs
    settings.default_user_agent = default_user_agent
    settings.default_referer = default_referer
    settings.default_accept_language = default_accept_language
    settings.nominatim_endpoint = nominatim_endpoint
    settings.nominatim_key = nominatim_key
    settings.overpass_endpoint = overpass_endpoint
    settings.all_oneway = all_oneway
    settings.elevation_provider = elevation_provider
    settings.bidirectional_network_types = bidirectional_network_types

    # if logging is turned on, log that we are configured
    if settings.log_file or settings.log_console:
        log("Configured osmnx")

def log(message, level=None, name=None, filename=None):
    """
    Write a message to the logger.

    This logs to file and/or prints to the console (terminal), depending on
    the current configuration of settings.log_file and settings.log_console.

    Parameters
    ----------
    message : string
        the message to log
    level : int
        one of the logger.level constants
    name : string
        name of the logger
    filename : string
        name of the log file

    Returns
    -------
    None
    """
    if level is None:
        level = settings.log_level
    if name is None:
        name = settings.log_name
    if filename is None:
        filename = settings.log_filename

    # if logging to file is turned on
    if settings.log_file:
        # get the current logger (or create a new one, if none), then log
        # message at requested level
        logger = _get_logger(level=level, name=name, filename=filename)
        if level == lg.DEBUG:
            logger.debug(message)
        elif level == lg.INFO:
            logger.info(message)
        elif level == lg.WARNING:
            logger.warning(message)
        elif level == lg.ERROR:
            logger.error(message)

    # if logging to console, convert message to ascii and print to console
    if settings.log_console:
        # capture current stdout, then switch it to the console, print the
        # message, then switch back to what had been the stdout. this prevents
        # logging to notebook in jupyter: instead, it goes to terminal
        standard_out = sys.stdout
        sys.stdout = sys.__stdout__

        # prepend timestamp
        message = f"{ts()} {message}"

        # convert to ascii so it doesn't break windows terminals
        message = (
            unicodedata.normalize("NFKD", str(message)).encode("ascii", errors="replace").decode()
        )
        print(message)
        sys.stdout = standard_out

def _get_logger(level=None, name=None, filename=None):
    """
    Create a logger or return the current one if already instantiated.

    Parameters
    ----------
    level : int
        one of the logger.level constants
    name : string
        name of the logger
    filename : string
        name of the log file

    Returns
    -------
    logger : logging.logger
    """
    if level is None:
        level = settings.log_level
    if name is None:
        name = settings.log_name
    if filename is None:
        filename = settings.log_filename

    logger = lg.getLogger(name)

    # if a logger with this name is not already set up
    if not getattr(logger, "handler_set", None):

        # get today's date and construct a log filename
        log_filename = os.path.join(settings.logs_folder, f'{filename}_{ts(style="date")}.log')

        # if the logs folder does not already exist, create it
        if not os.path.exists(settings.logs_folder):
            os.makedirs(settings.logs_folder)

        # create file handler and log formatter and set them up
        handler = lg.FileHandler(log_filename, encoding="utf-8")
        formatter = lg.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s")
        handler.setFormatter(formatter)
        logger.addHandler(handler)
        logger.setLevel(level)
        logger.handler_set = True

    return logger
<|endoftext|>


<commit_before># oxy.io
# File: oxyio/util/stats.py
# Desc: the stats system

import json
from datetime import datetime

from elasticquery import ElasticQuery, Query, Aggregate

from oxyio import settings
from oxyio.app import redis_client, es_client

STAT_TYPE_TO_AGG = {
    'total': Aggregate.sum,
    'fixed': Aggregate.avg
}

def get_stat_keys(type_=None, queries=None):
    '''Get unique stat keys.'''

    if queries is None:
        queries = []

    # Type filter?
    if type_:
        queries.append(Query.term('type', type_))

    # Build our query
    q = ElasticQuery(
        es=es_client,
        index=settings.ES_STATS_INDEX,
        doc_type='object_stat'
    )

    # Attach queries
    q.query(Query.and_(*queries))

    # Attach a terms aggregate
    q.aggregate(Aggregate.terms('key_terms', 'key'))

    # Get the results
    results = q.get()

    return [
        bucket['key']
        for bucket in results['aggregations']['key_terms']['buckets']
    ]

def get_stats(
    type_=None, keys=None, details=None, queries=None,
    stat_type='total', interval='1m', since='now-24h', to='now'
):
    '''Get stats from ES.'''

    if queries is None:
        queries = []

    # Always apply date range
    queries.append(Query.range('datetime', gte=since, lt=to))

    # Type filter?
    if type_:
        queries.append(Query.term('type', type_))

    # Top level date histogram aggregate
    top_aggregate = aggregate = Aggregate.date_histogram('dates', 'datetime', interval)

    # Build kwarg filters & aggregates
    for key, values in (
        ('key', keys),
        ('detail', details)
    ):
        # Handle term or terms queries
        if values:
            # If we're a list, we want to aggregate to split the data up between list
            # keys. Similarly True means we want all list keys.
            if isinstance(values, list) or values is True:
                query = Query.terms

                # Aggregate
                agg = Aggregate.terms('{0}_terms'.format(key), key)
                if aggregate:
                    aggregate.aggregate(agg)

                aggregate = agg

            else:
                query = Query.term

            # Filter if not getting all
            if values is not True:
                queries.append(query(key, values))

    # Attach the stat aggregate
    stat_agg = STAT_TYPE_TO_AGG.get(stat_type)
    stat_agg = stat_agg('value', 'value')

    # Nest aggregate
    if aggregate:
        aggregate.aggregate(stat_agg)

    # Assign nested stat agg as latest
    aggregate = stat_agg

    # Build our query
    q = ElasticQuery(
        es=es_client,
        index=settings.ES_STATS_INDEX,
        doc_type='object_stat'
    )

    # Attach queries
    q.query(Query.and_(*queries))

    # Add the original top level date histogram
    q.aggregate(top_aggregate)

    # Get results
    results = q.get()
    dates = []

    def parse_bucket(bucket):
        out = {}

        for field in ('key', 'detail'):
            terms_field = '{0}_terms'.format(field)

            if terms_field in bucket:
                out.update(parse_buckets(bucket[terms_field]['buckets']))
                out['key'] = bucket['key']
                break

        else:
            out = {
                'key': bucket['key'],
                'value': bucket['value']['value']
            }

        return out

    def parse_buckets(buckets):
        buckets = [parse_bucket(bucket) for bucket in buckets]

        return {
            bucket.pop('key'): bucket['value'] if 'value' in bucket else bucket
            for bucket in buckets
        }

    # Parse the results into appropriately nested list of dicts
    for bucket in results['aggregations']['dates']['buckets']:
        stats = {}

        stats['stats'] = parse_bucket(bucket)
        stats['stats'].pop('key')

        stats.update({
            'datetime': bucket['key_as_string']
        })

        dates.append(stats)

    return dates

def _object_function_wrapper(obj, handler, **kwargs):
    queries = kwargs.pop('queries', [])

    queries.extend([
        Query.term('object_module', obj.MODULE),
        Query.term('object_type', obj.OBJECT),
        Query.term('object_id', obj.id)
    ])

    return handler(queries=queries, **kwargs)

def get_object_stat_keys(obj, **kwargs):
    return _object_function_wrapper(obj, get_stat_keys, **kwargs)

def get_object_stats(obj, **kwargs):
    return _object_function_wrapper(obj, get_stats, **kwargs)

def index_object_stats(obj, stats, time=None):
    '''Push stats to the Redis index queue.'''

    iso_now = datetime.now().replace(microsecond=0).isoformat()

    for stat in stats:
        stat.update({
            'object_type': obj.OBJECT,
            'object_module': obj.MODULE,
            'object_id': obj.id,
            'datetime': iso_now
        })

        # Push onto the queue
        redis_client.lpush(
            settings.REDIS_INDEX_QUEUE,
            json.dumps(stat)
        )
<commit_msg>Elasticsearch v2 support.<commit_after># oxy.io
# File: oxyio/util/stats.py
# Desc: the stats system

import json
from datetime import datetime

from elasticquery import ElasticQuery, Query, Aggregate

from oxyio import settings
from oxyio.app import redis_client, es_client

STAT_TYPE_TO_AGG = {
    'total': Aggregate.sum,
    'fixed': Aggregate.avg
}

def get_stat_keys(type_=None, queries=None):
    '''Get unique stat keys.'''

    if queries is None:
        queries = []

    # Type filter?
    if type_:
        queries.append(Query.term('type', type_))

    # Build our query
    q = ElasticQuery(
        es=es_client,
        index=settings.ES_STATS_INDEX,
        doc_type='object_stat'
    )

    # Attach queries
    q.query(Query.bool(must=queries))

    # Attach a terms aggregate
    q.aggregate(Aggregate.terms('key_terms', 'key'))

    # Get the results
    results = q.get()

    return [
        bucket['key']
        for bucket in results['aggregations']['key_terms']['buckets']
    ]

def get_stats(
    type_=None, keys=None, details=None, queries=None,
    stat_type='total', interval='1m', since='now-24h', to='now'
):
    '''Get stats from ES.'''

    if queries is None:
        queries = []

    # Always apply date range
    queries.append(Query.range('datetime', gte=since, lt=to))

    # Type filter?
    if type_:
        queries.append(Query.term('type', type_))

    # Top level date histogram aggregate
    top_aggregate = aggregate = Aggregate.date_histogram('dates', 'datetime', interval)

    # Build kwarg filters & aggregates
    for key, values in (
        ('key', keys),
        ('detail', details)
    ):
        # Handle term or terms queries
        if values:
            # If we're a list, we want to aggregate to split the data up between list
            # keys. Similarly True means we want all list keys.
            if isinstance(values, list) or values is True:
                query = Query.terms

                # Aggregate
                agg = Aggregate.terms('{0}_terms'.format(key), key)
                if aggregate:
                    aggregate.aggregate(agg)

                aggregate = agg

            else:
                query = Query.term

            # Filter if not getting all
            if values is not True:
                queries.append(query(key, values))

    # Attach the stat aggregate
    stat_agg = STAT_TYPE_TO_AGG.get(stat_type)
    stat_agg = stat_agg('value', 'value')

    # Nest aggregate
    if aggregate:
        aggregate.aggregate(stat_agg)

    # Assign nested stat agg as latest
    aggregate = stat_agg

    # Build our query
    q = ElasticQuery(
        es=es_client,
        index=settings.ES_STATS_INDEX,
        doc_type='object_stat'
    )

    # Attach queries
    q.query(Query.bool(must=queries))

    # Add the original top level date histogram
    q.aggregate(top_aggregate)

    # Get results
    results = q.get()
    dates = []

    def parse_bucket(bucket):
        out = {}

        for field in ('key', 'detail'):
            terms_field = '{0}_terms'.format(field)

            if terms_field in bucket:
                out.update(parse_buckets(bucket[terms_field]['buckets']))
                out['key'] = bucket['key']
                break

        else:
            out = {
                'key': bucket['key'],
                'value': bucket['value']['value']
            }

        return out

    def parse_buckets(buckets):
        buckets = [parse_bucket(bucket) for bucket in buckets]

        return {
            bucket.pop('key'): bucket['value'] if 'value' in bucket else bucket
            for bucket in buckets
        }

    # Parse the results into appropriately nested list of dicts
    for bucket in results['aggregations']['dates']['buckets']:
        stats = {}

        stats['stats'] = parse_bucket(bucket)
        stats['stats'].pop('key')

        stats.update({
            'datetime': bucket['key_as_string']
        })

        dates.append(stats)

    return dates

def _object_function_wrapper(obj, handler, **kwargs):
    queries = kwargs.pop('queries', [])

    queries.extend([
        Query.term('object_module', obj.MODULE),
        Query.term('object_type', obj.OBJECT),
        Query.term('object_id', obj.id)
    ])

    return handler(queries=queries, **kwargs)

def get_object_stat_keys(obj, **kwargs):
    return _object_function_wrapper(obj, get_stat_keys, **kwargs)

def get_object_stats(obj, **kwargs):
    return _object_function_wrapper(obj, get_stats, **kwargs)

def index_object_stats(obj, stats, time=None):
    '''Push stats to the Redis index queue.'''

    iso_now = datetime.now().replace(microsecond=0).isoformat()

    for stat in stats:
        stat.update({
            'object_type': obj.OBJECT,
            'object_module': obj.MODULE,
            'object_id': obj.id,
            'datetime': iso_now
        })

        # Push onto the queue
        redis_client.lpush(
            settings.REDIS_INDEX_QUEUE,
            json.dumps(stat)
        )
<|endoftext|>


<commit_before>#
# CRIU API
# Includes class to work with CRIU service and helpers
#

import socket
import struct
import os
import rpc_pb2 as cr_rpc
import stats_pb2 as crs

criu_socket = "/var/run/criu_service.socket"
req_types = {
	cr_rpc.DUMP: "dump",
	cr_rpc.PRE_DUMP: "pre_dump",
	cr_rpc.PAGE_SERVER: "page_server",
	cr_rpc.RESTORE: "restore"
}

def_verb = 2

#
# Connection to CRIU service
#

class criu_conn:
	def __enter__(self):
		self.cs = socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET)
		self.cs.connect(criu_socket)
		self.verb = def_verb
		return self

	def __exit__(self, type, value, traceback):
		self.cs.close()

	def verbose(self, level):
		self.verb = level

	def send_req(self, req, with_resp = True):
		req.opts.log_level = self.verb
		req.opts.log_file = "criu_%s.log" % req_types[req.type]
		self.cs.send(req.SerializeToString())
		if with_resp:
			return self.recv_resp()

	def recv_resp(self):
		resp = cr_rpc.criu_resp()
		resp.ParseFromString(self.cs.recv(1024))
		return resp

	def ack_notify(self, success = True):
		req = cr_rpc.criu_req()
		req.type = cr_rpc.NOTIFY
		req.notify_success = True
		self.cs.send(req.SerializeToString())

#
# Helper to read CRIU-generated statistics
#

CRIU_STATS_MAGIC = 0x57093306

def criu_get_stats(img, file_name):
	s = struct.Struct("I I")
	f = open(os.path.join(img.work_dir(), file_name))
	#
	# Stats file is 4 butes of magic, then 4 bytes with
	# stats packet size
	#
	v = s.unpack(f.read(s.size))
	if v[0] != CRIU_STATS_MAGIC:
		raise Exception("Magic is %x, expect %x" % (v[0], CRIU_STATS_MAGIC))

	stats = crs.stats_entry()
	stats.ParseFromString(f.read(v[1]))

	return stats

def criu_get_dstats(img):
	stats = criu_get_stats(img, "stats-dump")
	return stats.dump

def criu_get_rstats(img):
	stats = criu_get_stats(img, "stats-restore")
	return stats.restore
<commit_msg>criu: Use swrk instead of remote service<commit_after>#
# CRIU API
# Includes class to work with CRIU service and helpers
#

import socket
import struct
import os
import subprocess
import rpc_pb2 as cr_rpc
import stats_pb2 as crs

criu_binary = "/root/criu/criu"

req_types = {
	cr_rpc.DUMP: "dump",
	cr_rpc.PRE_DUMP: "pre_dump",
	cr_rpc.PAGE_SERVER: "page_server",
	cr_rpc.RESTORE: "restore"
}

def_verb = 2

#
# Connection to CRIU service
#

class criu_conn:
	def __enter__(self):
		css = socket.socketpair(socket.AF_UNIX, socket.SOCK_SEQPACKET)
		self.swrk = subprocess.Popen([criu_binary, "swrk", "%d" % css[0].fileno()])
		css[0].close()
		self.cs = css[1]
		return self

	def __exit__(self, type, value, traceback):
		self.cs.close()
		self.swrk.wait()

	def verbose(self, level):
		self.verb = level

	def send_req(self, req, with_resp = True):
		req.opts.log_level = self.verb
		req.opts.log_file = "criu_%s.log" % req_types[req.type]
		self.cs.send(req.SerializeToString())
		if with_resp:
			return self.recv_resp()

	def recv_resp(self):
		resp = cr_rpc.criu_resp()
		resp.ParseFromString(self.cs.recv(1024))
		return resp

	def ack_notify(self, success = True):
		req = cr_rpc.criu_req()
		req.type = cr_rpc.NOTIFY
		req.notify_success = True
		self.cs.send(req.SerializeToString())

#
# Helper to read CRIU-generated statistics
#

CRIU_STATS_MAGIC = 0x57093306

def criu_get_stats(img, file_name):
	s = struct.Struct("I I")
	f = open(os.path.join(img.work_dir(), file_name))
	#
	# Stats file is 4 butes of magic, then 4 bytes with
	# stats packet size
	#
	v = s.unpack(f.read(s.size))
	if v[0] != CRIU_STATS_MAGIC:
		raise Exception("Magic is %x, expect %x" % (v[0], CRIU_STATS_MAGIC))

	stats = crs.stats_entry()
	stats.ParseFromString(f.read(v[1]))

	return stats

def criu_get_dstats(img):
	stats = criu_get_stats(img, "stats-dump")
	return stats.dump

def criu_get_rstats(img):
	stats = criu_get_stats(img, "stats-restore")
	return stats.restore
<|endoftext|>


<commit_before>#!/usr/bin/env python

#    Copyright 2014 Mikko Korpela
#
#    This file is part of Pabot - A parallel executor for Robot Framework test cases..
#
#    Pabot is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    Pabot is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with Pabot.  If not, see <http://www.gnu.org/licenses/>.
#
#  partly based on work by Nokia Solutions and Networks Oyj
#  that was licensed under Apache License Version 2.0

import os, sys, time, datetime
import multiprocessing
from glob import glob
from StringIO import StringIO
import shutil
import subprocess
from robot import run, rebot
from robot import __version__ as ROBOT_VERSION
from robot.api import ExecutionResult
from robot.errors import Information
from robot.result.visitor import ResultVisitor
from multiprocessing.pool import ThreadPool
from robot.run import USAGE
from robot.utils import ArgumentParser
import signal
from result_merger import merge

CTRL_C_PRESSED = False

def execute_and_wait_with(args):
    global CTRL_C_PRESSED
    if CTRL_C_PRESSED:
        # Keyboard interrupt has happened!
        return
    time.sleep(0)
    datasources, outs_dir, options, suite_name, command, verbose = args
    datasources = [d.encode('utf-8') if isinstance(d, unicode) else d for d in datasources]
    outs_dir = os.path.join(outs_dir, suite_name)
    cmd = command + _options_for_custom_executor(options, outs_dir, suite_name) + datasources
    cmd = [c if ' ' not in c else '"%s"' % c for c in cmd]
    os.makedirs(outs_dir)
    with open(os.path.join(outs_dir, 'stdout.txt'), 'w') as stdout, \
        open(os.path.join(outs_dir, 'stderr.txt'), 'w') as stderr:
            process, rc = _run(cmd, stderr, stdout, suite_name, verbose)
    if rc != 0:
        print _execution_failed_message(suite_name, process, rc, verbose)
    else:
        print 'PASSED %s' % suite_name

def _run(cmd, stderr, stdout, suite_name, verbose):
    process = subprocess.Popen(' '.join(cmd),
                               shell=True,
                               stderr=stderr,
                               stdout=stdout)
    if verbose:
        print '[PID:%s] EXECUTING PARALLEL SUITE %s with command:\n%s' % (process.pid, suite_name, ' '.join(cmd))
    else:
        print '[PID:%s] EXECUTING %s' % (process.pid, suite_name)
    rc = None
    while rc is None:
        rc = process.poll()
        time.sleep(0.1)
    return process, rc

def _execution_failed_message(suite_name, process, rc, verbose):
    if not verbose:
        return 'FAILED %s' % suite_name
    return 'Execution failed in %s with %d failing test(s)' % (suite_name, rc)

def _options_for_custom_executor(*args):
    return _options_to_cli_arguments(_options_for_executor(*args))

def _options_for_executor(options, outs_dir, suite_name):
    options = options.copy()
    options['log'] = 'NONE'
    options['report'] = 'NONE'
    options['xunit'] = 'NONE'
    options['suite'] = suite_name
    options['outputdir'] = outs_dir
    options['monitorcolors'] = 'off'
    if ROBOT_VERSION >= '2.8':
        options['monitormarkers'] = 'off'
    return options

def _options_to_cli_arguments(opts):
    res = []
    for k, v in opts.items():
        if isinstance(v, str):
            res += ['--' + str(k), str(v)]
        elif isinstance(v, unicode):
            res += ['--' + str(k), v.encode('utf-8')]
        elif isinstance(v, bool) and (v is True):
            res += ['--' + str(k)]
        elif isinstance(v, list):
            for value in v:
                if isinstance(value, unicode):
                    res += ['--' + str(k), value.encode('utf-8')]
                else:
                    res += ['--' + str(k), str(value)]
    return res

class GatherSuiteNames(ResultVisitor):

      def __init__(self):
          self.result = []

      def end_suite(self, suite):
          if len(suite.tests):
             self.result.append(suite.longname)

def get_suite_names(output_file):
    if not os.path.isfile(output_file):
       return []
    try:
       e = ExecutionResult(output_file)
       gatherer = GatherSuiteNames()
       e.visit(gatherer)
       return gatherer.result
    except:
       return []

def _parse_args(args):
    pabot_args = {'command':['pybot'],
                  'verbose':False,
                  'processes':max(multiprocessing.cpu_count(), 2)}
    while args and args[0] in ['--command', '--processes', '--verbose']:
        if args[0] == '--command':
            end_index = args.index('--end-command')
            pabot_args['command'] = args[1:end_index]
            args = args[end_index+1:]
        if args[0] == '--processes':
            pabot_args['processes'] = int(args[1])
            args = args[2:]
        if args[0] == '--verbose':
            pabot_args['verbose'] = True
            args = args[1:]
    options, datasources = ArgumentParser(USAGE, auto_pythonpath=False, auto_argumentfile=False).parse_args(args)
    keys = set()
    for k in options:
        if options[k] is None:
            keys.add(k)
    for k in keys:
        del options[k]
    return options, datasources, pabot_args

def solve_suite_names(outs_dir, datasources, options):
    opts = _options_for_dryrun(options, outs_dir)
    run(*datasources, **opts)
    output = os.path.join(outs_dir, opts['output'])
    suite_names = get_suite_names(output)
    if os.path.isfile(output):
        os.remove(output)
    return suite_names

def _options_for_dryrun(options, outs_dir):
    options = options.copy()
    options['log'] = 'NONE'
    options['report'] = 'NONE'
    options['xunit'] = 'NONE'
    if ROBOT_VERSION >= '2.8':
        options['dryrun'] = True
    else:
        options['runmode'] = 'DryRun'
    options['output'] = 'suite_names.xml'
    options['outputdir'] = outs_dir
    options['stdout'] = StringIO()
    options['stderr'] = StringIO()
    options['monitorcolors'] = 'off'
    if ROBOT_VERSION >= '2.8':
        options['monitormarkers'] = 'off'
    return options

def _options_for_rebot(options, start_time_string, end_time_string):
    rebot_options = options.copy()
    rebot_options['starttime'] = start_time_string
    rebot_options['endtime'] = end_time_string
    rebot_options['monitorcolors'] = 'off'
    if ROBOT_VERSION >= '2.8':
        options['monitormarkers'] = 'off'
    return rebot_options

def _now():
    return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')

def _print_elapsed(start, end):
    elapsed = end - start
    millis = int((elapsed * 1000) % 1000)
    seconds = int(elapsed) % 60
    elapsed_minutes = (int(elapsed)-seconds)/60
    minutes = elapsed_minutes % 60
    elapsed_hours = (elapsed_minutes-minutes)/60
    elapsed_string = ''
    if elapsed_hours > 0:
        elapsed_string += '%d hours ' % elapsed_hours
    elapsed_string += '%d minutes %d.%d seconds' % (minutes, seconds, millis)
    print 'Elapsed time: '+elapsed_string

def keyboard_interrupt(*args):
    global CTRL_C_PRESSED
    CTRL_C_PRESSED = True

def _parallel_execute(datasources, options, outs_dir, pabot_args, suite_names):
    if suite_names:
        original_signal_handler = signal.signal(signal.SIGINT, keyboard_interrupt)
        pool = ThreadPool(pabot_args['processes'])
        result = pool.map_async(execute_and_wait_with,
                   [(datasources,
                     outs_dir,
                     options,
                     suite,
                     pabot_args['command'],
                     pabot_args['verbose'])
                    for suite in suite_names])
        pool.close()
        while not result.ready():
            # keyboard interrupt is executed in main thread and needs this loop to get time to get executed
            try:
                time.sleep(0.1)
            except IOError:
                keyboard_interrupt()
        signal.signal(signal.SIGINT, original_signal_handler)

def _output_dir(options):
    outputdir = '.'
    if 'outputdir' in options:
        outputdir = options['outputdir']
    outpath = os.path.join(outputdir, 'pabot_results')
    if os.path.isdir(outpath):
        shutil.rmtree(outpath)
    return outpath

def _report_results(outs_dir, options, start_time_string):
    output_path = os.path.abspath(os.path.join(options.get('outputdir', '.'), options.get('output', 'output.xml')))
    merge(*sorted(glob(os.path.join(outs_dir, '**/*.xml')))).save(output_path)
    print 'Output:  %s' % output_path
    options['output'] = None # Do not write output again with rebot
    return rebot(output_path, **_options_for_rebot(options, start_time_string, _now()))

def main(args):
    start_time = time.time()
    start_time_string = _now()
    #NOTE: timeout option
    try:
        options, datasources, pabot_args = _parse_args(args)
        outs_dir = _output_dir(options)
        suite_names = solve_suite_names(outs_dir, datasources, options)
        _parallel_execute(datasources, options, outs_dir, pabot_args, suite_names)
        sys.exit(_report_results(outs_dir, options, start_time_string))
    except Information, i:
        print """A parallel executor for Robot Framework test cases.

Supports all Robot Framework command line options and also following options (these must be before normal RF options):

--verbose
more output

--command [ACTUAL COMMANDS TO START ROBOT EXECUTOR] --end-command
RF script for situations where pybot is not used directly

--processes [NUMBER OF PROCESSES]
How many parallel executors to use (default max of 2 and cpu count)

Copyright 2014 Mikko Korpela - GPLv3
"""
        print i.message
    finally:
        _print_elapsed(start_time, time.time())

if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>message writer from threads<commit_after>#!/usr/bin/env python

#    Copyright 2014 Mikko Korpela
#
#    This file is part of Pabot - A parallel executor for Robot Framework test cases..
#
#    Pabot is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    Pabot is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with Pabot.  If not, see <http://www.gnu.org/licenses/>.
#
#  partly based on work by Nokia Solutions and Networks Oyj
#  that was licensed under Apache License Version 2.0

import os, sys, time, datetime
import multiprocessing
from glob import glob
from StringIO import StringIO
import shutil
import subprocess
import threading
from robot import run, rebot
from robot import __version__ as ROBOT_VERSION
from robot.api import ExecutionResult
from robot.errors import Information
from robot.result.visitor import ResultVisitor
from multiprocessing.pool import ThreadPool
from robot.run import USAGE
from robot.utils import ArgumentParser
import signal
from result_merger import merge
import Queue

CTRL_C_PRESSED = False
MESSAGE_QUEUE = Queue.Queue()

def execute_and_wait_with(args):
    global CTRL_C_PRESSED
    if CTRL_C_PRESSED:
        # Keyboard interrupt has happened!
        return
    time.sleep(0)
    datasources, outs_dir, options, suite_name, command, verbose = args
    datasources = [d.encode('utf-8') if isinstance(d, unicode) else d for d in datasources]
    outs_dir = os.path.join(outs_dir, suite_name)
    cmd = command + _options_for_custom_executor(options, outs_dir, suite_name) + datasources
    cmd = [c if ' ' not in c else '"%s"' % c for c in cmd]
    os.makedirs(outs_dir)
    with open(os.path.join(outs_dir, 'stdout.txt'), 'w') as stdout, \
        open(os.path.join(outs_dir, 'stderr.txt'), 'w') as stderr:
            process, rc = _run(cmd, stderr, stdout, suite_name, verbose)
    if rc != 0:
        _write(_execution_failed_message(suite_name, process, rc, verbose))
    else:
        _write('PASSED %s' % suite_name)

def _run(cmd, stderr, stdout, suite_name, verbose):
    process = subprocess.Popen(' '.join(cmd),
                               shell=True,
                               stderr=stderr,
                               stdout=stdout)
    if verbose:
        _write('[PID:%s] EXECUTING PARALLEL SUITE %s with command:\n%s' % (process.pid, suite_name, ' '.join(cmd)))
    else:
        _write('[PID:%s] EXECUTING %s' % (process.pid, suite_name))
    rc = None
    elapsed = 0
    while rc is None:
        rc = process.poll()
        time.sleep(0.1)
        elapsed += 1
        if elapsed % 150 == 0:
            _write('[PID:%s] still running %s after %s seconds' % (process.pid, suite_name, elapsed / 10.0))
    return process, rc

def _execution_failed_message(suite_name, process, rc, verbose):
    if not verbose:
        return 'FAILED %s' % suite_name
    return 'Execution failed in %s with %d failing test(s)' % (suite_name, rc)

def _options_for_custom_executor(*args):
    return _options_to_cli_arguments(_options_for_executor(*args))

def _options_for_executor(options, outs_dir, suite_name):
    options = options.copy()
    options['log'] = 'NONE'
    options['report'] = 'NONE'
    options['xunit'] = 'NONE'
    options['suite'] = suite_name
    options['outputdir'] = outs_dir
    options['monitorcolors'] = 'off'
    if ROBOT_VERSION >= '2.8':
        options['monitormarkers'] = 'off'
    return options

def _options_to_cli_arguments(opts):
    res = []
    for k, v in opts.items():
        if isinstance(v, str):
            res += ['--' + str(k), str(v)]
        elif isinstance(v, unicode):
            res += ['--' + str(k), v.encode('utf-8')]
        elif isinstance(v, bool) and (v is True):
            res += ['--' + str(k)]
        elif isinstance(v, list):
            for value in v:
                if isinstance(value, unicode):
                    res += ['--' + str(k), value.encode('utf-8')]
                else:
                    res += ['--' + str(k), str(value)]
    return res

class GatherSuiteNames(ResultVisitor):

      def __init__(self):
          self.result = []

      def end_suite(self, suite):
          if len(suite.tests):
             self.result.append(suite.longname)

def get_suite_names(output_file):
    if not os.path.isfile(output_file):
       return []
    try:
       e = ExecutionResult(output_file)
       gatherer = GatherSuiteNames()
       e.visit(gatherer)
       return gatherer.result
    except:
       return []

def _parse_args(args):
    pabot_args = {'command':['pybot'],
                  'verbose':False,
                  'processes':max(multiprocessing.cpu_count(), 2)}
    while args and args[0] in ['--command', '--processes', '--verbose']:
        if args[0] == '--command':
            end_index = args.index('--end-command')
            pabot_args['command'] = args[1:end_index]
            args = args[end_index+1:]
        if args[0] == '--processes':
            pabot_args['processes'] = int(args[1])
            args = args[2:]
        if args[0] == '--verbose':
            pabot_args['verbose'] = True
            args = args[1:]
    options, datasources = ArgumentParser(USAGE, auto_pythonpath=False, auto_argumentfile=False).parse_args(args)
    keys = set()
    for k in options:
        if options[k] is None:
            keys.add(k)
    for k in keys:
        del options[k]
    return options, datasources, pabot_args

def solve_suite_names(outs_dir, datasources, options):
    opts = _options_for_dryrun(options, outs_dir)
    run(*datasources, **opts)
    output = os.path.join(outs_dir, opts['output'])
    suite_names = get_suite_names(output)
    if os.path.isfile(output):
        os.remove(output)
    return suite_names

def _options_for_dryrun(options, outs_dir):
    options = options.copy()
    options['log'] = 'NONE'
    options['report'] = 'NONE'
    options['xunit'] = 'NONE'
    if ROBOT_VERSION >= '2.8':
        options['dryrun'] = True
    else:
        options['runmode'] = 'DryRun'
    options['output'] = 'suite_names.xml'
    options['outputdir'] = outs_dir
    options['stdout'] = StringIO()
    options['stderr'] = StringIO()
    options['monitorcolors'] = 'off'
    if ROBOT_VERSION >= '2.8':
        options['monitormarkers'] = 'off'
    return options

def _options_for_rebot(options, start_time_string, end_time_string):
    rebot_options = options.copy()
    rebot_options['starttime'] = start_time_string
    rebot_options['endtime'] = end_time_string
    rebot_options['monitorcolors'] = 'off'
    if ROBOT_VERSION >= '2.8':
        options['monitormarkers'] = 'off'
    return rebot_options

def _now():
    return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')

def _print_elapsed(start, end):
    elapsed = end - start
    millis = int((elapsed * 1000) % 1000)
    seconds = int(elapsed) % 60
    elapsed_minutes = (int(elapsed)-seconds)/60
    minutes = elapsed_minutes % 60
    elapsed_hours = (elapsed_minutes-minutes)/60
    elapsed_string = ''
    if elapsed_hours > 0:
        elapsed_string += '%d hours ' % elapsed_hours
    elapsed_string += '%d minutes %d.%d seconds' % (minutes, seconds, millis)
    print 'Elapsed time: '+elapsed_string

def keyboard_interrupt(*args):
    global CTRL_C_PRESSED
    CTRL_C_PRESSED = True

def _parallel_execute(datasources, options, outs_dir, pabot_args, suite_names):
    if suite_names:
        original_signal_handler = signal.signal(signal.SIGINT, keyboard_interrupt)
        pool = ThreadPool(pabot_args['processes'])
        result = pool.map_async(execute_and_wait_with,
                   [(datasources,
                     outs_dir,
                     options,
                     suite,
                     pabot_args['command'],
                     pabot_args['verbose'])
                    for suite in suite_names])
        pool.close()
        while not result.ready():
            # keyboard interrupt is executed in main thread and needs this loop to get time to get executed
            try:
                time.sleep(0.1)
            except IOError:
                keyboard_interrupt()
        signal.signal(signal.SIGINT, original_signal_handler)

def _output_dir(options):
    outputdir = '.'
    if 'outputdir' in options:
        outputdir = options['outputdir']
    outpath = os.path.join(outputdir, 'pabot_results')
    if os.path.isdir(outpath):
        shutil.rmtree(outpath)
    return outpath

def _report_results(outs_dir, options, start_time_string):
    output_path = os.path.abspath(os.path.join(options.get('outputdir', '.'), options.get('output', 'output.xml')))
    merge(*sorted(glob(os.path.join(outs_dir, '**/*.xml')))).save(output_path)
    print 'Output:  %s' % output_path
    options['output'] = None # Do not write output again with rebot
    return rebot(output_path, **_options_for_rebot(options, start_time_string, _now()))

def _writer():
    while True:
        message = MESSAGE_QUEUE.get()
        print message

def _write(message):
    MESSAGE_QUEUE.put(message)

def _start_message_writer():
    t = threading.Thread(target=_writer)
    t.setDaemon(True)
    t.start()

def main(args):
    start_time = time.time()
    start_time_string = _now()
    #NOTE: timeout option
    try:
        _start_message_writer()
        options, datasources, pabot_args = _parse_args(args)
        outs_dir = _output_dir(options)
        suite_names = solve_suite_names(outs_dir, datasources, options)
        _parallel_execute(datasources, options, outs_dir, pabot_args, suite_names)
        sys.exit(_report_results(outs_dir, options, start_time_string))
    except Information, i:
        print """A parallel executor for Robot Framework test cases.

Supports all Robot Framework command line options and also following options (these must be before normal RF options):

--verbose
more output

--command [ACTUAL COMMANDS TO START ROBOT EXECUTOR] --end-command
RF script for situations where pybot is not used directly

--processes [NUMBER OF PROCESSES]
How many parallel executors to use (default max of 2 and cpu count)

Copyright 2014 Mikko Korpela - GPLv3
"""
        print i.message
    finally:
        _print_elapsed(start_time, time.time())

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>


<commit_before>ELECTRUM_VERSION = "2.3.1"  # version of the client package
PROTOCOL_VERSION = '0.9'    # protocol version requested
NEW_SEED_VERSION = 11       # electrum versions >= 2.0
OLD_SEED_VERSION = 4        # electrum versions < 2.0

# The hash of the mnemonic seed must begin with this
SEED_PREFIX      = '01'      # Electrum standard wallet
SEED_PREFIX_2FA  = '101'     # extended seed for two-factor authentication
<commit_msg>Litecoin: bump version to 2.3.1.1<commit_after>ELECTRUM_VERSION = "2.3.1.1" # version of the client package
PROTOCOL_VERSION = '0.9'    # protocol version requested
NEW_SEED_VERSION = 11       # electrum versions >= 2.0
OLD_SEED_VERSION = 4        # electrum versions < 2.0

# The hash of the mnemonic seed must begin with this
SEED_PREFIX      = '01'      # Electrum standard wallet
SEED_PREFIX_2FA  = '101'     # extended seed for two-factor authentication
<|endoftext|>


<commit_before>#!/usr/bin/env python

#  Copyright 2014 Mikko Korpela
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
#  partly based on work by Nokia Solutions and Networks Oyj

import re
import os, sys, time, datetime
import multiprocessing
from glob import glob
from StringIO import StringIO
import shutil
import subprocess
import threading
from robot import run, rebot
from robot import __version__ as ROBOT_VERSION
from robot.api import ExecutionResult
from robot.errors import Information
from robot.result.visitor import ResultVisitor
from robot.libraries.Remote import Remote
from multiprocessing.pool import ThreadPool
from robot.run import USAGE
from robot.utils import ArgumentParser
import signal
import PabotLib
from result_merger import merge
import Queue

CTRL_C_PRESSED = False
MESSAGE_QUEUE = Queue.Queue()
_PABOTLIBURI = '127.0.0.1:8270'

def execute_and_wait_with(args):
    global CTRL_C_PRESSED
    if CTRL_C_PRESSED:
        # Keyboard interrupt has happened!
        return
    time.sleep(0)
    datasources, outs_dir, options, suite_name, command, verbose = args
    datasources = [d.encode('utf-8') if isinstance(d, unicode) else d for d in datasources]
    outs_dir = os.path.join(outs_dir, suite_name)
    cmd = command + _options_for_custom_executor(options, outs_dir, suite_name) + datasources
    cmd = [c if ' ' not in c else '"%s"' % c for c in cmd]
    os.makedirs(outs_dir)
    with open(os.path.join(outs_dir, 'stdout.txt'), 'w') as stdout:
        with open(os.path.join(outs_dir, 'stderr.txt'), 'w') as stderr:
            process, rc = _run(cmd, stderr, stdout, suite_name, verbose)
    if rc != 0:
        _write(_execution_failed_message(suite_name, rc, verbose))
    else:
        _write('PASSED %s' % suite_name)

def _run(cmd, stderr, stdout, suite_name, verbose):
    process = subprocess.Popen(' '.join(cmd),
                               shell=True,
                               stderr=stderr,
                               stdout=stdout)
    if verbose:
        _write('[PID:%s] EXECUTING PARALLEL SUITE %s with command:\n%s' % (process.pid, suite_name, ' '.join(cmd)))
    else:
        _write('[PID:%s] EXECUTING %s' % (process.pid, suite_name))
    rc = None
    elapsed = 0
    while rc is None:
        rc = process.poll()
        time.sleep(0.1)
        elapsed += 1
        if elapsed % 150 == 0:
            _write('[PID:%s] still running %s after %s seconds' % (process.pid, suite_name, elapsed / 10.0))
    return process, rc

def _execution_failed_message(suite_name, rc, verbose):
    if not verbose:
        return 'FAILED %s' % suite_name
    return 'Execution failed in %s with %d failing test(s)' % (suite_name, rc)

def _options_for_custom_executor(*args):
    return _options_to_cli_arguments(_options_for_executor(*args))

def _options_for_executor(options, outs_dir, suite_name):
    options = options.copy()
    options['log'] = 'NONE'
    options['report'] = 'NONE'
    options['xunit'] = 'NONE'
    options['suite'] = suite_name
    options['outputdir'] = outs_dir
    options['monitorcolors'] = 'off'
    options['variable'] = options.get('variable')
    options['variable'].append('PABOTLIBURI:%s' % _PABOTLIBURI)
    if ROBOT_VERSION >= '2.8':
        options['monitormarkers'] = 'off'
    return options

def _options_to_cli_arguments(opts):
    res = []
    for k, v in opts.items():
        if isinstance(v, str):
            res += ['--' + str(k), str(v)]
        elif isinstance(v, unicode):
            res += ['--' + str(k), v.encode('utf-8')]
        elif isinstance(v, bool) and (v is True):
            res += ['--' + str(k)]
        elif isinstance(v, list):
            for value in v:
                if isinstance(value, unicode):
                    res += ['--' + str(k), value.encode('utf-8')]
                else:
                    res += ['--' + str(k), str(value)]
    return res

class GatherSuiteNames(ResultVisitor):

      def __init__(self):
          self.result = []

      def end_suite(self, suite):
          if len(suite.tests):
             self.result.append(suite.longname)

def get_suite_names(output_file):
    if not os.path.isfile(output_file):
       return []
    try:
       e = ExecutionResult(output_file)
       gatherer = GatherSuiteNames()
       e.visit(gatherer)
       return gatherer.result
    except:
       return []

def _parse_args(args):
    pabot_args = {'command':['pybot'],
                  'verbose':False,
                  'pabotlib':False,
                  'processes':max(multiprocessing.cpu_count(), 2)}
    while args and args[0] in ['--'+param for param in ['command', 'processes', 'verbose', 'resourcefile',
                                                        'pabotlib']]:
        if args[0] == '--command':
            end_index = args.index('--end-command')
            pabot_args['command'] = args[1:end_index]
            args = args[end_index+1:]
        if args[0] == '--processes':
            pabot_args['processes'] = int(args[1])
            args = args[2:]
        if args[0] == '--verbose':
            pabot_args['verbose'] = True
            args = args[1:]
        if args[0] == '--resourcefile':
            pabot_args['resourcefile'] = args[1]
            args = args[2:]
        if args[0] == '--pabotlib':
            pabot_args['pabotlib'] = True
            args = args[1:]
    options, datasources = ArgumentParser(USAGE, auto_pythonpath=False, auto_argumentfile=False).parse_args(args)
    keys = set()
    for k in options:
        if options[k] is None:
            keys.add(k)
    for k in keys:
        del options[k]
    return options, datasources, pabot_args

def solve_suite_names(outs_dir, datasources, options):
    opts = _options_for_dryrun(options, outs_dir)
    run(*datasources, **opts)
    output = os.path.join(outs_dir, opts['output'])
    suite_names = get_suite_names(output)
    if os.path.isfile(output):
        os.remove(output)
    return suite_names

def _options_for_dryrun(options, outs_dir):
    options = options.copy()
    options['log'] = 'NONE'
    options['report'] = 'NONE'
    options['xunit'] = 'NONE'
    if ROBOT_VERSION >= '2.8':
        options['dryrun'] = True
    else:
        options['runmode'] = 'DryRun'
    options['output'] = 'suite_names.xml'
    options['outputdir'] = outs_dir
    options['stdout'] = StringIO()
    options['stderr'] = StringIO()
    options['monitorcolors'] = 'off'
    if ROBOT_VERSION >= '2.8':
        options['monitormarkers'] = 'off'
    return options

def _options_for_rebot(options, start_time_string, end_time_string):
    rebot_options = options.copy()
    rebot_options['starttime'] = start_time_string
    rebot_options['endtime'] = end_time_string
    rebot_options['monitorcolors'] = 'off'
    if ROBOT_VERSION >= '2.8':
        options['monitormarkers'] = 'off'
    return rebot_options

def _now():
    return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')

def _print_elapsed(start, end):
    elapsed = end - start
    millis = int((elapsed * 1000) % 1000)
    seconds = int(elapsed) % 60
    elapsed_minutes = (int(elapsed)-seconds)/60
    minutes = elapsed_minutes % 60
    elapsed_hours = (elapsed_minutes-minutes)/60
    elapsed_string = ''
    if elapsed_hours > 0:
        elapsed_string += '%d hours ' % elapsed_hours
    elapsed_string += '%d minutes %d.%d seconds' % (minutes, seconds, millis)
    print 'Elapsed time: '+elapsed_string

def keyboard_interrupt(*args):
    global CTRL_C_PRESSED
    CTRL_C_PRESSED = True

def _parallel_execute(datasources, options, outs_dir, pabot_args, suite_names):
    original_signal_handler = signal.signal(signal.SIGINT, keyboard_interrupt)
    pool = ThreadPool(pabot_args['processes'])
    result = pool.map_async(execute_and_wait_with,
               [(datasources,
                 outs_dir,
                 options,
                 suite,
                 pabot_args['command'],
                 pabot_args['verbose'])
                for suite in suite_names])
    pool.close()
    while not result.ready():
        # keyboard interrupt is executed in main thread and needs this loop to get time to get executed
        try:
            time.sleep(0.1)
        except IOError:
            keyboard_interrupt()
    signal.signal(signal.SIGINT, original_signal_handler)

def _output_dir(options, cleanup=True):
    outputdir = options.get('outputdir', '.')
    outpath = os.path.join(outputdir, 'pabot_results')
    if cleanup and os.path.isdir(outpath):
        shutil.rmtree(outpath)
    return outpath

def _copy_screenshots(options):
    pabot_outputdir = _output_dir(options, cleanup=False)
    outputdir = options.get('outputdir', '.')
    for location, dir_names, file_names in os.walk(pabot_outputdir):
        for file_name in file_names:
            if re.search("selenium-screenshot-.*\.png", file_name):
                prefix = os.path.relpath(location, pabot_outputdir)
                dst_file_name = '-'.join([prefix, file_name])
                shutil.copyfile(os.path.join(location, file_name),
                                os.path.join(outputdir, dst_file_name))

def _report_results(outs_dir, options, start_time_string):
    output_path = os.path.abspath(os.path.join(options.get('outputdir', '.'), options.get('output', 'output.xml')))
    merge(*sorted(glob(os.path.join(outs_dir, '**/*.xml')))).save(output_path)
    _copy_screenshots(options)
    print 'Output:  %s' % output_path
    options['output'] = None # Do not write output again with rebot
    return rebot(output_path, **_options_for_rebot(options, start_time_string, _now()))

def _writer():
    while True:
        message = MESSAGE_QUEUE.get()
        print message

def _write(message):
    MESSAGE_QUEUE.put(message)

def _start_message_writer():
    t = threading.Thread(target=_writer)
    t.setDaemon(True)
    t.start()

def _start_remote_library(pabot_args):
    if not pabot_args['pabotlib']:
        return None
    return subprocess.Popen('python %s %s' % (os.path.abspath(PabotLib.__file__),
                                              pabot_args.get('resourcefile', 'N/A')),
                            shell=True)

def _stop_remote_library(process):
    print 'Stopping PabotLib process'
    Remote(_PABOTLIBURI).run_keyword('stop_remote_server', [], {})
    i = 50
    while i > 0 and process.poll() is None:
        time.sleep(0.1)
        i -= 1
    if i == 0:
        print 'Could not stop PabotLib Process in 5 seconds - calling terminate'
        process.terminate()
    else:
        print 'PabotLib process stopped'

def _set_top_name(suite_names):
    top_names = [x.split('.')[0] for x in suite_names]
    if top_names.count(top_names[0]) == len(top_names):
        os.environ['TESTS_TOP_NAME'] = top_names[0]

def main(args):
    start_time = time.time()
    start_time_string = _now()
    lib_process = None
    #NOTE: timeout option
    try:
        _start_message_writer()
        options, datasources, pabot_args = _parse_args(args)
        lib_process = _start_remote_library(pabot_args)
        outs_dir = _output_dir(options)
        suite_names = solve_suite_names(outs_dir, datasources, options)
        if suite_names:
            _set_top_name(suite_names)
            _parallel_execute(datasources, options, outs_dir, pabot_args, suite_names)
            sys.exit(_report_results(outs_dir, options, start_time_string))
        else:
            print 'No tests to execute'
    except Information, i:
        print """A parallel executor for Robot Framework test cases. Version 0.14.

Supports all Robot Framework command line options and also following options (these must be before normal RF options):

--verbose
more output

--command [ACTUAL COMMANDS TO START ROBOT EXECUTOR] --end-command
RF script for situations where pybot is not used directly

--processes [NUMBER OF PROCESSES]
How many parallel executors to use (default max of 2 and cpu count)

--resourcefile [FILEPATH]
Indicator for a file that can contain shared variables for distributing resources.

--pabotlib
Start PabotLib remote server. This enables locking and resource distribution between parallel test executions.

Copyright 2015 Mikko Korpela - Apache 2 License
"""
        print i.message
    finally:
        if lib_process:
            _stop_remote_library(lib_process)
        _print_elapsed(start_time, time.time())

if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>Passed/Failed messages are colored in the console.<commit_after>#!/usr/bin/env python

#  Copyright 2014 Mikko Korpela
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
#  partly based on work by Nokia Solutions and Networks Oyj

import re
import os, sys, time, datetime
import multiprocessing
from glob import glob
from StringIO import StringIO
import shutil
import subprocess
import threading
from robot import run, rebot
from robot import __version__ as ROBOT_VERSION
from robot.api import ExecutionResult
from robot.errors import Information
from robot.result.visitor import ResultVisitor
from robot.libraries.Remote import Remote
from multiprocessing.pool import ThreadPool
from robot.run import USAGE
from robot.utils import ArgumentParser
import signal
import PabotLib
from result_merger import merge
import Queue

CTRL_C_PRESSED = False
MESSAGE_QUEUE = Queue.Queue()
_PABOTLIBURI = '127.0.0.1:8270'

class Color:
    GREEN = '\033[92m'
    RED = '\033[91m'
    ENDC = '\033[0m'

def execute_and_wait_with(args):
    global CTRL_C_PRESSED
    if CTRL_C_PRESSED:
        # Keyboard interrupt has happened!
        return
    time.sleep(0)
    datasources, outs_dir, options, suite_name, command, verbose = args
    datasources = [d.encode('utf-8') if isinstance(d, unicode) else d for d in datasources]
    outs_dir = os.path.join(outs_dir, suite_name)
    cmd = command + _options_for_custom_executor(options, outs_dir, suite_name) + datasources
    cmd = [c if ' ' not in c else '"%s"' % c for c in cmd]
    os.makedirs(outs_dir)
    with open(os.path.join(outs_dir, 'stdout.txt'), 'w') as stdout:
        with open(os.path.join(outs_dir, 'stderr.txt'), 'w') as stderr:
            process, rc = _run(cmd, stderr, stdout, suite_name, verbose)
    if rc != 0:
        _write(_execution_failed_message(suite_name, rc, verbose), Color.RED)
    else:
        _write('PASSED %s' % suite_name, Color.GREEN)

def _run(cmd, stderr, stdout, suite_name, verbose):
    process = subprocess.Popen(' '.join(cmd),
                               shell=True,
                               stderr=stderr,
                               stdout=stdout)
    if verbose:
        _write('[PID:%s] EXECUTING PARALLEL SUITE %s with command:\n%s' % (process.pid, suite_name, ' '.join(cmd)))
    else:
        _write('[PID:%s] EXECUTING %s' % (process.pid, suite_name))
    rc = None
    elapsed = 0
    while rc is None:
        rc = process.poll()
        time.sleep(0.1)
        elapsed += 1
        if elapsed % 150 == 0:
            _write('[PID:%s] still running %s after %s seconds' % (process.pid, suite_name, elapsed / 10.0))
    return process, rc

def _execution_failed_message(suite_name, rc, verbose):
    if not verbose:
        return 'FAILED %s' % suite_name
    return 'Execution failed in %s with %d failing test(s)' % (suite_name, rc)

def _options_for_custom_executor(*args):
    return _options_to_cli_arguments(_options_for_executor(*args))

def _options_for_executor(options, outs_dir, suite_name):
    options = options.copy()
    options['log'] = 'NONE'
    options['report'] = 'NONE'
    options['xunit'] = 'NONE'
    options['suite'] = suite_name
    options['outputdir'] = outs_dir
    options['monitorcolors'] = 'off'
    options['variable'] = options.get('variable')
    options['variable'].append('PABOTLIBURI:%s' % _PABOTLIBURI)
    if ROBOT_VERSION >= '2.8':
        options['monitormarkers'] = 'off'
    return options

def _options_to_cli_arguments(opts):
    res = []
    for k, v in opts.items():
        if isinstance(v, str):
            res += ['--' + str(k), str(v)]
        elif isinstance(v, unicode):
            res += ['--' + str(k), v.encode('utf-8')]
        elif isinstance(v, bool) and (v is True):
            res += ['--' + str(k)]
        elif isinstance(v, list):
            for value in v:
                if isinstance(value, unicode):
                    res += ['--' + str(k), value.encode('utf-8')]
                else:
                    res += ['--' + str(k), str(value)]
    return res

class GatherSuiteNames(ResultVisitor):

      def __init__(self):
          self.result = []

      def end_suite(self, suite):
          if len(suite.tests):
             self.result.append(suite.longname)

def get_suite_names(output_file):
    if not os.path.isfile(output_file):
       return []
    try:
       e = ExecutionResult(output_file)
       gatherer = GatherSuiteNames()
       e.visit(gatherer)
       return gatherer.result
    except:
       return []

def _parse_args(args):
    pabot_args = {'command':['pybot'],
                  'verbose':False,
                  'pabotlib':False,
                  'processes':max(multiprocessing.cpu_count(), 2)}
    while args and args[0] in ['--'+param for param in ['command', 'processes', 'verbose', 'resourcefile',
                                                        'pabotlib']]:
        if args[0] == '--command':
            end_index = args.index('--end-command')
            pabot_args['command'] = args[1:end_index]
            args = args[end_index+1:]
        if args[0] == '--processes':
            pabot_args['processes'] = int(args[1])
            args = args[2:]
        if args[0] == '--verbose':
            pabot_args['verbose'] = True
            args = args[1:]
        if args[0] == '--resourcefile':
            pabot_args['resourcefile'] = args[1]
            args = args[2:]
        if args[0] == '--pabotlib':
            pabot_args['pabotlib'] = True
            args = args[1:]
    options, datasources = ArgumentParser(USAGE, auto_pythonpath=False, auto_argumentfile=False).parse_args(args)
    keys = set()
    for k in options:
        if options[k] is None:
            keys.add(k)
    for k in keys:
        del options[k]
    return options, datasources, pabot_args

def solve_suite_names(outs_dir, datasources, options):
    opts = _options_for_dryrun(options, outs_dir)
    run(*datasources, **opts)
    output = os.path.join(outs_dir, opts['output'])
    suite_names = get_suite_names(output)
    if os.path.isfile(output):
        os.remove(output)
    return suite_names

def _options_for_dryrun(options, outs_dir):
    options = options.copy()
    options['log'] = 'NONE'
    options['report'] = 'NONE'
    options['xunit'] = 'NONE'
    if ROBOT_VERSION >= '2.8':
        options['dryrun'] = True
    else:
        options['runmode'] = 'DryRun'
    options['output'] = 'suite_names.xml'
    options['outputdir'] = outs_dir
    options['stdout'] = StringIO()
    options['stderr'] = StringIO()
    options['monitorcolors'] = 'off'
    if ROBOT_VERSION >= '2.8':
        options['monitormarkers'] = 'off'
    return options

def _options_for_rebot(options, start_time_string, end_time_string):
    rebot_options = options.copy()
    rebot_options['starttime'] = start_time_string
    rebot_options['endtime'] = end_time_string
    rebot_options['monitorcolors'] = 'off'
    if ROBOT_VERSION >= '2.8':
        options['monitormarkers'] = 'off'
    return rebot_options

def _now():
    return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')

def _print_elapsed(start, end):
    elapsed = end - start
    millis = int((elapsed * 1000) % 1000)
    seconds = int(elapsed) % 60
    elapsed_minutes = (int(elapsed)-seconds)/60
    minutes = elapsed_minutes % 60
    elapsed_hours = (elapsed_minutes-minutes)/60
    elapsed_string = ''
    if elapsed_hours > 0:
        elapsed_string += '%d hours ' % elapsed_hours
    elapsed_string += '%d minutes %d.%d seconds' % (minutes, seconds, millis)
    print 'Elapsed time: '+elapsed_string

def keyboard_interrupt(*args):
    global CTRL_C_PRESSED
    CTRL_C_PRESSED = True

def _parallel_execute(datasources, options, outs_dir, pabot_args, suite_names):
    original_signal_handler = signal.signal(signal.SIGINT, keyboard_interrupt)
    pool = ThreadPool(pabot_args['processes'])
    result = pool.map_async(execute_and_wait_with,
               [(datasources,
                 outs_dir,
                 options,
                 suite,
                 pabot_args['command'],
                 pabot_args['verbose'])
                for suite in suite_names])
    pool.close()
    while not result.ready():
        # keyboard interrupt is executed in main thread and needs this loop to get time to get executed
        try:
            time.sleep(0.1)
        except IOError:
            keyboard_interrupt()
    signal.signal(signal.SIGINT, original_signal_handler)

def _output_dir(options, cleanup=True):
    outputdir = options.get('outputdir', '.')
    outpath = os.path.join(outputdir, 'pabot_results')
    if cleanup and os.path.isdir(outpath):
        shutil.rmtree(outpath)
    return outpath

def _copy_screenshots(options):
    pabot_outputdir = _output_dir(options, cleanup=False)
    outputdir = options.get('outputdir', '.')
    for location, dir_names, file_names in os.walk(pabot_outputdir):
        for file_name in file_names:
            if re.search("selenium-screenshot-.*\.png", file_name):
                prefix = os.path.relpath(location, pabot_outputdir)
                dst_file_name = '-'.join([prefix, file_name])
                shutil.copyfile(os.path.join(location, file_name),
                                os.path.join(outputdir, dst_file_name))

def _report_results(outs_dir, options, start_time_string):
    output_path = os.path.abspath(os.path.join(options.get('outputdir', '.'), options.get('output', 'output.xml')))
    merge(*sorted(glob(os.path.join(outs_dir, '**/*.xml')))).save(output_path)
    _copy_screenshots(options)
    print 'Output:  %s' % output_path
    options['output'] = None # Do not write output again with rebot
    return rebot(output_path, **_options_for_rebot(options, start_time_string, _now()))

def _writer():
    while True:
        message = MESSAGE_QUEUE.get()
        print message

def _write(message, color=None):
    if color == Color.RED:
        message = _wrap_with_red(message)
    if color == Color.GREEN:
        message = _wrap_with_green(message)
    MESSAGE_QUEUE.put(message)

def _wrap_with_red(text):
    return "%s%s%s" % (Color.RED, text, Color.ENDC)

def _wrap_with_green(text):
    return "%s%s%s" % (Color.GREEN, text, Color.ENDC)

def _start_message_writer():
    t = threading.Thread(target=_writer)
    t.setDaemon(True)
    t.start()

def _start_remote_library(pabot_args):
    if not pabot_args['pabotlib']:
        return None
    return subprocess.Popen('python %s %s' % (os.path.abspath(PabotLib.__file__),
                                              pabot_args.get('resourcefile', 'N/A')),
                            shell=True)

def _stop_remote_library(process):
    print 'Stopping PabotLib process'
    Remote(_PABOTLIBURI).run_keyword('stop_remote_server', [], {})
    i = 50
    while i > 0 and process.poll() is None:
        time.sleep(0.1)
        i -= 1
    if i == 0:
        print 'Could not stop PabotLib Process in 5 seconds - calling terminate'
        process.terminate()
    else:
        print 'PabotLib process stopped'

def _set_top_name(suite_names):
    top_names = [x.split('.')[0] for x in suite_names]
    if top_names.count(top_names[0]) == len(top_names):
        os.environ['TESTS_TOP_NAME'] = top_names[0]

def main(args):
    start_time = time.time()
    start_time_string = _now()
    lib_process = None
    #NOTE: timeout option
    try:
        _start_message_writer()
        options, datasources, pabot_args = _parse_args(args)
        lib_process = _start_remote_library(pabot_args)
        outs_dir = _output_dir(options)
        suite_names = solve_suite_names(outs_dir, datasources, options)
        if suite_names:
            _set_top_name(suite_names)
            _parallel_execute(datasources, options, outs_dir, pabot_args, suite_names)
            sys.exit(_report_results(outs_dir, options, start_time_string))
        else:
            print 'No tests to execute'
    except Information, i:
        print """A parallel executor for Robot Framework test cases. Version 0.14.

Supports all Robot Framework command line options and also following options (these must be before normal RF options):

--verbose
more output

--command [ACTUAL COMMANDS TO START ROBOT EXECUTOR] --end-command
RF script for situations where pybot is not used directly

--processes [NUMBER OF PROCESSES]
How many parallel executors to use (default max of 2 and cpu count)

--resourcefile [FILEPATH]
Indicator for a file that can contain shared variables for distributing resources.

--pabotlib
Start PabotLib remote server. This enables locking and resource distribution between parallel test executions.

Copyright 2015 Mikko Korpela - Apache 2 License
"""
        print i.message
    finally:
        if lib_process:
            _stop_remote_library(lib_process)
        _print_elapsed(start_time, time.time())

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>


<commit_before>"""
Django settings for example project.

Generated by 'django-admin startproject' using Django 1.9.6.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '*ymubzn8p_s7vrm%jsqvr6$qnea_5mcp(ao0z-yh1q0gro!0g1'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',
    'django_filters',
    'rest_framework',
    'djadmin2',
    'djadmin2.tests',
    'djadmin2.themes.djadmin2theme_bootstrap3',
    'blog',
    'files',
    'polls'
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'example.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [os.path.join(BASE_DIR, 'templates')],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'example.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

SITE_ID = 1

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    os.path.join(BASE_DIR, "static"),
)

MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
MEDIA_URL = "/media/"

REST_FRAMEWORK = {
    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination',
    'PAGE_SIZE': 10
}
ADMIN2_THEME_DIRECTORY = "djadmin2theme_bootstrap3"<commit_msg>settings changes<commit_after>"""
Django settings for example project.

Generated by 'django-admin startproject' using Django 1.9.6.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

os.environ['DJANGO_SETTINGS_MODULE'] = 'example.settings'

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '*ymubzn8p_s7vrm%jsqvr6$qnea_5mcp(ao0z-yh1q0gro!0g1'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',
    'django_filters',
    'rest_framework',
    'djadmin2',
    'djadmin2.tests',
    'djadmin2.themes.djadmin2theme_bootstrap3',
    'blog',
    'files',
    'polls'
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'example.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [os.path.join(BASE_DIR, 'templates')],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'example.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

SITE_ID = 1

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    os.path.join(BASE_DIR, "static"),
)

MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
MEDIA_URL = "/media/"

REST_FRAMEWORK = {
    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination',
    'PAGE_SIZE': 10,

    'DEFAULT_PERMISSION_CLASSES': [
        'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly'
    ]
}

ADMIN2_THEME_DIRECTORY = "djadmin2theme_bootstrap3"
<|endoftext|>