<commit_before>"""
Renaming tests. This means search for related names.
I always leave a little bit of space to add room for additions, because the
results always contain position informations.
"""
#< 4 (0,4), (3,0), (5,0)
def abc(): pass

#< 0 (-3,4), (0,0), (2,0)
abc.d.a.bsaasd.abc.d

abc

abc =

#< (-3,0), (0,0)
abc

Abc = 3

#< 6 (0,6), (2,4), (5,8), (17,0)
class Abc():
    #< (-2,6), (0,4), (3,8), (15,0)
    Abc

    def Abc(self):
        Abc; self.c = 3

    #< 17 (0,16), (2,8)
    def a(self, Abc):
        #< 10 (-2,16), (0,8)
        Abc

    #< 19 (0,18), (2,8)
    def self_test(self):
        #< 12 (-2,18), (0,8)
        self.b

Abc.d.Abc

#< 4 (0,4), (4,1)
def blub():

#< (-4,4), (0,1)
@blub
def a(): pass

#< (0,7), (3,0)
import module_not_exists

#< (-3,7), (0,0)
module_not_exists

#< ('rename1', 1,0), (0,24), (3,0), (6,17), ('rename2', 4,5), (10,17), (13,17)
from import_tree import rename1

#< (0,8), ('rename1',3,0), ('rename2',4,20), ('rename2',6,0), (3,32), (7,32), (4,0)
rename1.abc

#< (-3,8), ('rename1', 3,0), ('rename2', 4,20), ('rename2', 6,0), (0,32), (4,32), (1,0)
from import_tree.rename1 import abc
abc

#< 20 ('rename1', 1,0), ('rename2', 4,5), (-10,24), (-7,0), (-4,17), (0,17), (3,17)
from import_tree.rename1 import abc

#< (0, 32),
from import_tree.rename1 import not_existing

response = 5
#< 0 (0,0), (1,0), (2,0), (4,0)
response = HttpResponse(mimetype='application/pdf')
response['Content-Disposition'] = 'attachment; filename=%s.pdf' % id
response.write(pdf)
#< (-4,0), (-3,0), (-2,0), (0,0)
response
<commit_msg>removed whitespace in renaming tests<commit_after>"""
Renaming tests. This means search for related names.
I always leave a little bit of space to add room for additions, because the
results always contain position informations.
"""
#< 4 (0,4), (3,0), (5,0)
def abc(): pass

#< 0 (-3,4), (0,0), (2,0)
abc.d.a.bsaasd.abc.d

abc

abc =

#< (-3,0), (0,0)
abc

Abc = 3

#< 6 (0,6), (2,4), (5,8), (17,0)
class Abc():
    #< (-2,6), (0,4), (3,8), (15,0)
    Abc

    def Abc(self):
        Abc; self.c = 3

    #< 17 (0,16), (2,8)
    def a(self, Abc):
        #< 10 (-2,16), (0,8)
        Abc

    #< 19 (0,18), (2,8)
    def self_test(self):
        #< 12 (-2,18), (0,8)
        self.b

Abc.d.Abc

#< 4 (0,4), (4,1)
def blub():

#< (-4,4), (0,1)
@blub
def a(): pass

#< (0,7), (3,0)
import module_not_exists

#< (-3,7), (0,0)
module_not_exists

#< ('rename1', 1,0), (0,24), (3,0), (6,17), ('rename2', 4,5), (10,17), (13,17)
from import_tree import rename1

#< (0,8), ('rename1',3,0), ('rename2',4,20), ('rename2',6,0), (3,32), (7,32), (4,0)
rename1.abc

#< (-3,8), ('rename1', 3,0), ('rename2', 4,20), ('rename2', 6,0), (0,32), (4,32), (1,0)
from import_tree.rename1 import abc
abc

#< 20 ('rename1', 1,0), ('rename2', 4,5), (-10,24), (-7,0), (-4,17), (0,17), (3,17)
from import_tree.rename1 import abc

#< (0, 32),
from import_tree.rename1 import not_existing

response = 5
#< 0 (0,0), (1,0), (2,0), (4,0)
response = HttpResponse(mimetype='application/pdf')
response['Content-Disposition'] = 'attachment; filename=%s.pdf' % id
response.write(pdf)
#< (-4,0), (-3,0), (-2,0), (0,0)
response
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Arne Neumann

"""
The ``paula`` module converts a ``DiscourseDocumentGraph`` (possibly
containing multiple annotation layers) into a PAULA XML document. Our goal is
to produce the subset of the PAULA 'specification' that is understood by the
SaltNPepper converter framework

"""

from collections import defaultdict

from lxml import etree
from lxml.builder import ElementMaker

from discoursegraphs import (EdgeTypes, get_text, get_top_level_layers,
                             select_edges_by, tokens2text)
from discoursegraphs.util import create_dir, natural_sort_key

NSMAP={'xlink': 'http://www.w3.org/1999/xlink',
       'xml': 'http://www.w3.org/XML/1998/namespace'}

IGNORED_TOKEN_FEATURES = ('layers', 'label', 'tiger:token', 'tiger:word',
                          'tiger:id')

class PaulaDocument(object):
    """
    This class converts a DiscourseDocumentGraph into a PAULA XML document
    (i.e. a set of XML files describing one document annotated on multiple
    lebels).
    """
    def __init__(self, docgraph, corpus_name='mycorpus', human_readable=False):
        """
        Parameters
        ----------
        docgraph : DiscourseDocumentGraph
            the document graph to be converted
        corpus_name : str
            name of the corpus this document belongs to
        """
        self.corpus_name = corpus_name
        self.E = ElementMaker()
        self.files = defaultdict(str)  # map file types to file names

        self.primary_text = self.__gen_primary_text_file(docgraph)
        self.tokenization = self.__gen_tokenization_file(docgraph)
        self.token_annotation = \
            self.__gen_token_annotation_file(docgraph,
                                             human_readable=human_readable)

        self.span_markable_files = []
        self.hierarchy_files = []
        for top_level_layer in get_top_level_layers(docgraph):
            self.span_markable_files.append(
                self.__gen_span_markables_file(docgraph, top_level_layer,
                                               human_readable=human_readable))

    def __gen_primary_text_file(self, docgraph):
        """
        generate the PAULA file that contains the primary text of the document
        graph.

        Example
        -------
        <?xml version="1.0" standalone="no"?>
        <!DOCTYPE paula SYSTEM "paula_text.dtd">
        <paula version="1.1">

            Zum Angewöhnen ...
        </paula>
        """
        tree = self.E('paula', version='1.1')
        tree.append(self.E('header', paula_id='{}.text'.format(docgraph.name)))
        body = self.E('body')
        primary_text = get_text(docgraph)
        body.text = primary_text
        tree.append(body)
        return tree

    def __gen_tokenization_file(self, docgraph):
        """
        generate the PAULA file that contains the tokenization of the document
        graph.

        Example
        -------
        <?xml version="1.0" standalone="no"?>
        <!DOCTYPE paula SYSTEM "paula_mark.dtd">
        <paula version="1.1">

            <markList xmlns:xlink="http://www.w3.org/1999/xlink" type="tok" xml:base="maz-1423.text.xml">

                ...
            </markList>
        </paula>
        ...
        """
        paula_id = '{}.{}.tok'.format(self.corpus_name, docgraph.name)
        self.files['tokenization'] = paula_id+'.xml'

        E = ElementMaker(nsmap=NSMAP)
        tree = E('paula', version='1.1')
        tree.append(E('header', paula_id=paula_id))
        mlist = E('markList', {'type': 'tok',
                               '{%s}base' % NSMAP['xml']: '{}.{}.text.xml'.format(self.corpus_name, docgraph.name)})
        tok_tuples = docgraph.get_tokens()
        for (tid, onset, tlen) in get_onsets(tok_tuples):
            xp = "#xpointer(string-range(//body,'',{},{}))".format(onset, tlen)
            mlist.append(E('mark', {'id': tid,
                                    '{%s}href' % NSMAP['xlink']: xp}))
        tree.append(mlist)
        return tree

    def etree_to_string(self, tree):
        return etree.tostring(tree, pretty_print=True, xml_declaration=True,
                              encoding="UTF-8",
                              doctype='<!DOCTYPE paula SYSTEM "paula_text.dtd">',
                              standalone='no',
                              xml_version='1.0')

    def __gen_span_markables_file(self, docgraph, layer, human_readable=True):
        """
        """
        E = ElementMaker(nsmap=NSMAP)
        tree = E('paula', version='1.1')
        tree.append(E('header', paula_id='{}.{}_{}_seg'.format(self.corpus_name, docgraph.name, layer)))
        mlist = E('markList', {'type': 'tok',
                               '{%s}base' % NSMAP['xml']: '{}.{}.tok.xml'.format(self.corpus_name, docgraph.name)})

        span_dict = defaultdict(lambda : defaultdict(str))
        edges = select_edges_by(docgraph, layer=layer,
                                edge_type=EdgeTypes.spanning_relation,
                                data=True)
        for source_id, target_id, edge_attrs in edges:
            span_dict[source_id][target_id] = edge_attrs

        target_dict = defaultdict(list)
        for source_id in span_dict:
            target_ids = sorted(span_dict[source_id], key=natural_sort_key)
            xp = '(#xpointer(id({})/range-to(id({}))))'.format(target_ids[0],
                                                               target_ids[-1])
            mark = E('mark', {'{%s}href' % NSMAP['xlink']: xp})
            if human_readable:
                # add  containing the token strings
                mark.append(etree.Comment(tokens2text(docgraph, target_ids)))
                target_dict[target_ids[0]].append(mark)
            else:
                mlist.append(mark)

        if human_readable:  # order  elements by token ordering
            for target in sorted(target_dict, key=natural_sort_key):
                for mark in target_dict[target]:
                    mlist.append(mark)

        tree.append(mlist)
        return tree

    def __gen_token_annotation_file(self, docgraph, human_readable=True):
        """
        creates an etree representation of a multiFeat file that describes all
        the annotations that only span one token (e.g. POS, lemma etc.)
        """
        basefile = '{}.{}.tok.xml'.format(self.corpus_name, docgraph.name)
        paula_id = '{}.{}.tok_multiFeat'.format(self.corpus_name,
                                                docgraph.name)
        E = ElementMaker(nsmap=NSMAP)
        tree = E('paula', version='1.1')
        tree.append(E('header', paula_id=paula_id))
        mflist = E('multiFeatList', {'{%s}base' % NSMAP['xml']: basefile})

        for token_id in docgraph.tokens:
            mfeat = E('multiFeat',
                      {'{%s}href' % NSMAP['xlink']: '#{}'.format(token_id)})
            token_dict = docgraph.node[token_id]
            for feature in token_dict:
                if feature not in IGNORED_TOKEN_FEATURES:
                    mfeat.append(
                        E('feat',
                          {'name': feature, 'value': token_dict[feature]}))
            if human_readable:  # adds token string as a
                mfeat.append(etree.Comment(token_dict[docgraph.ns+':token']))
            mflist.append(mfeat)
        tree.append(mflist)
        return tree

    def write(self, output_rootdir):
        """
        Parameters
        ----------
        output_rootdir : str
            in the output root directory, a directory (with the name of the
            document ID) will be created. This document directory will contain
            all the annotations in PAULA XML format.

        """
        create_dir(output_rootdir)
        raise NotImplementedError

def get_onsets(token_tuples):
    """
    Parameters
    ----------
    token_tuples : list of (str, unicode)
        a list/generator of (token ID, token string) tuples

    Returns
    -------
    onset_tuples : generator of (str, int, int)
        A list/generator of (token ID, onset, token length) tuples.
        Note that PAULA starts counting string onsets with 1!
    """
    onset = 1
    for (token_id, token) in token_tuples:
        yield (token_id, onset, len(token))
        onset += (len(token) + 1)
<commit_msg>minor: moved method<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Arne Neumann

"""
The ``paula`` module converts a ``DiscourseDocumentGraph`` (possibly
containing multiple annotation layers) into a PAULA XML document. Our goal is
to produce the subset of the PAULA 'specification' that is understood by the
SaltNPepper converter framework

"""

from collections import defaultdict

from lxml import etree
from lxml.builder import ElementMaker

from discoursegraphs import (EdgeTypes, get_text, get_top_level_layers,
                             select_edges_by, tokens2text)
from discoursegraphs.util import create_dir, natural_sort_key

NSMAP={'xlink': 'http://www.w3.org/1999/xlink',
       'xml': 'http://www.w3.org/XML/1998/namespace'}

IGNORED_TOKEN_FEATURES = ('layers', 'label', 'tiger:token', 'tiger:word',
                          'tiger:id')

class PaulaDocument(object):
    """
    This class converts a DiscourseDocumentGraph into a PAULA XML document
    (i.e. a set of XML files describing one document annotated on multiple
    lebels).
    """
    def __init__(self, docgraph, corpus_name='mycorpus', human_readable=False):
        """
        Parameters
        ----------
        docgraph : DiscourseDocumentGraph
            the document graph to be converted
        corpus_name : str
            name of the corpus this document belongs to
        """
        self.corpus_name = corpus_name
        self.E = ElementMaker()
        self.files = defaultdict(str)  # map file types to file names

        self.primary_text = self.__gen_primary_text_file(docgraph)
        self.tokenization = self.__gen_tokenization_file(docgraph)
        self.token_annotation = \
            self.__gen_token_annotation_file(docgraph,
                                             human_readable=human_readable)

        self.span_markable_files = []
        self.hierarchy_files = []
        for top_level_layer in get_top_level_layers(docgraph):
            self.span_markable_files.append(
                self.__gen_span_markables_file(docgraph, top_level_layer,
                                               human_readable=human_readable))

    def __gen_primary_text_file(self, docgraph):
        """
        generate the PAULA file that contains the primary text of the document
        graph.

        Example
        -------
        <?xml version="1.0" standalone="no"?>
        <!DOCTYPE paula SYSTEM "paula_text.dtd">
        <paula version="1.1">

            Zum Angewöhnen ...
        </paula>
        """
        tree = self.E('paula', version='1.1')
        tree.append(self.E('header', paula_id='{}.text'.format(docgraph.name)))
        body = self.E('body')
        primary_text = get_text(docgraph)
        body.text = primary_text
        tree.append(body)
        return tree

    def __gen_tokenization_file(self, docgraph):
        """
        generate the PAULA file that contains the tokenization of the document
        graph.

        Example
        -------
        <?xml version="1.0" standalone="no"?>
        <!DOCTYPE paula SYSTEM "paula_mark.dtd">
        <paula version="1.1">

            <markList xmlns:xlink="http://www.w3.org/1999/xlink" type="tok" xml:base="maz-1423.text.xml">

                ...
            </markList>
        </paula>
        ...
        """
        paula_id = '{}.{}.tok'.format(self.corpus_name, docgraph.name)
        self.files['tokenization'] = paula_id+'.xml'

        E = ElementMaker(nsmap=NSMAP)
        tree = E('paula', version='1.1')
        tree.append(E('header', paula_id=paula_id))
        mlist = E('markList', {'type': 'tok',
                               '{%s}base' % NSMAP['xml']: '{}.{}.text.xml'.format(self.corpus_name, docgraph.name)})
        tok_tuples = docgraph.get_tokens()
        for (tid, onset, tlen) in get_onsets(tok_tuples):
            xp = "#xpointer(string-range(//body,'',{},{}))".format(onset, tlen)
            mlist.append(E('mark', {'id': tid,
                                    '{%s}href' % NSMAP['xlink']: xp}))
        tree.append(mlist)
        return tree

    def __gen_span_markables_file(self, docgraph, layer, human_readable=True):
        """
        """
        E = ElementMaker(nsmap=NSMAP)
        tree = E('paula', version='1.1')
        tree.append(E('header', paula_id='{}.{}_{}_seg'.format(self.corpus_name, docgraph.name, layer)))
        mlist = E('markList', {'type': 'tok',
                               '{%s}base' % NSMAP['xml']: '{}.{}.tok.xml'.format(self.corpus_name, docgraph.name)})

        span_dict = defaultdict(lambda : defaultdict(str))
        edges = select_edges_by(docgraph, layer=layer,
                                edge_type=EdgeTypes.spanning_relation,
                                data=True)
        for source_id, target_id, edge_attrs in edges:
            span_dict[source_id][target_id] = edge_attrs

        target_dict = defaultdict(list)
        for source_id in span_dict:
            target_ids = sorted(span_dict[source_id], key=natural_sort_key)
            xp = '(#xpointer(id({})/range-to(id({}))))'.format(target_ids[0],
                                                               target_ids[-1])
            mark = E('mark', {'{%s}href' % NSMAP['xlink']: xp})
            if human_readable:
                # add  containing the token strings
                mark.append(etree.Comment(tokens2text(docgraph, target_ids)))
                target_dict[target_ids[0]].append(mark)
            else:
                mlist.append(mark)

        if human_readable:  # order  elements by token ordering
            for target in sorted(target_dict, key=natural_sort_key):
                for mark in target_dict[target]:
                    mlist.append(mark)

        tree.append(mlist)
        return tree

    def __gen_token_annotation_file(self, docgraph, human_readable=True):
        """
        creates an etree representation of a multiFeat file that describes all
        the annotations that only span one token (e.g. POS, lemma etc.)
        """
        basefile = '{}.{}.tok.xml'.format(self.corpus_name, docgraph.name)
        paula_id = '{}.{}.tok_multiFeat'.format(self.corpus_name,
                                                docgraph.name)
        E = ElementMaker(nsmap=NSMAP)
        tree = E('paula', version='1.1')
        tree.append(E('header', paula_id=paula_id))
        mflist = E('multiFeatList', {'{%s}base' % NSMAP['xml']: basefile})

        for token_id in docgraph.tokens:
            mfeat = E('multiFeat',
                      {'{%s}href' % NSMAP['xlink']: '#{}'.format(token_id)})
            token_dict = docgraph.node[token_id]
            for feature in token_dict:
                if feature not in IGNORED_TOKEN_FEATURES:
                    mfeat.append(
                        E('feat',
                          {'name': feature, 'value': token_dict[feature]}))
            if human_readable:  # adds token string as a
                mfeat.append(etree.Comment(token_dict[docgraph.ns+':token']))
            mflist.append(mfeat)
        tree.append(mflist)
        return tree

    def etree_to_string(self, tree):
        return etree.tostring(tree, pretty_print=True, xml_declaration=True,
                              encoding="UTF-8",
                              doctype='<!DOCTYPE paula SYSTEM "paula_text.dtd">',
                              standalone='no',
                              xml_version='1.0')

    def write(self, output_rootdir):
        """
        Parameters
        ----------
        output_rootdir : str
            in the output root directory, a directory (with the name of the
            document ID) will be created. This document directory will contain
            all the annotations in PAULA XML format.

        """
        create_dir(output_rootdir)
        raise NotImplementedError

def get_onsets(token_tuples):
    """
    Parameters
    ----------
    token_tuples : list of (str, unicode)
        a list/generator of (token ID, token string) tuples

    Returns
    -------
    onset_tuples : generator of (str, int, int)
        A list/generator of (token ID, onset, token length) tuples.
        Note that PAULA starts counting string onsets with 1!
    """
    onset = 1
    for (token_id, token) in token_tuples:
        yield (token_id, onset, len(token))
        onset += (len(token) + 1)
<|endoftext|>


<commit_before># Source Server Stats
# File: sourcestats/util/elastic.py
# Desc: elasticsearch utilities

from datetime import datetime, timedelta

from flask import request, g
from elasticsearch import Elasticsearch
from elasticquery import ElasticQuery, Aggregate, Filter

from sourcestats import settings

from .request import in_request_args

ES_CLIENT = None
DEFAULT_INTERVAL = settings.COLLECT_INTERVAL / 60
SINCE_INTERVALS = {
    'd': 'days',
    'h': 'hours'
}

def get_es_client():
    global ES_CLIENT

    if ES_CLIENT is None:
        ES_CLIENT = Elasticsearch(settings.ES_HOSTS)

    return ES_CLIENT

def get_es_query(index=settings.SERVERS_INDEX):
    return ElasticQuery(
        es=get_es_client(),
        index=index,
        doc_type='server'
    )

def _get_since():
    if hasattr(g, 'request_since'):
        return g.request_since

    since = None

    # Attempt to parse any since
    if in_request_args('since'):
        interval = request.args['since'][-1]

        if interval in SINCE_INTERVALS:
            try:
                value = int(request.args['since'][0:-1])
                since = datetime.utcnow() - timedelta(**{
                    SINCE_INTERVALS[interval]: value
                })
            except ValueError:
                pass

    # Default to a day
    if since is None:
        since = datetime.utcnow() - timedelta(days=1)

    # ES mapping format has no microsecond
    since = since.replace(microsecond=0)

    # Cache for current request & return
    g.request_since = since
    return since

def _past_time(**kwargs):
    return (datetime.utcnow() - timedelta(**kwargs)).replace(microsecond=0)

def get_request_interval():
    since = _get_since()

    # If we're under a day, every 5 mins, like the collector
    if since >= _past_time(days=1):
        return DEFAULT_INTERVAL

    # If we're under one week, every 30m
    elif since >= _past_time(days=7):
        return 30

    # If we're under one month, every 2 hours
    elif since >= _past_time(days=31):
        return 120

    # Default a day
    return 1440

def get_request_filters():
    filters = []

    since = _get_since()
    filters.append(Filter.range('datetime', gte=since))

    for field in [
        'game_id',
        'map'
    ]:
        if in_request_args(field):
            filters.append(Filter.term(field, request.args[field]))

    return filters

def get_es_terms(field_name, filters=None, size=None, index=settings.SERVERS_INDEX):
    '''List 'objects', ie distinct field values from the indexes.'''
    q = get_es_query(index=index)
    q.size(0)

    if size is None:
        size = settings.ES_TERMS

    if filters:
        q.filter(Filter.and_(*filters))

    # Aggregates to generate terms and distinct values
    aggregates = (
        Aggregate.terms('objects', field_name, size=size),
        Aggregate.cardinality('values', field_name, precision_threshold=10000)
    )

    nested = False

    # Nested aggregate
    if '.' in field_name:
        nested_field, field_name = field_name.split('.')
        nested_aggregate = Aggregate.nested('nested', nested_field).aggregate(*aggregates)
        q.aggregate(nested_aggregate)
        nested = True

    # Normal aggregate
    else:
        q.aggregate(*aggregates)

    results = q.get()
    aggregations = results['aggregations']

    if nested:
        results_object = aggregations['nested']['objects']['buckets']
        total = aggregations['nested']['values']['value']
    else:
        results_object = aggregations['objects']['buckets']
        total = aggregations['values']['value']

    # For history docs the doc_counts in terms are useless as they'll just be duplicates
    if index == settings.HISTORY_INDEXES:
        objects = [bucket['key'] for bucket in results_object]

    # But in servers they represent what we see live
    else:
        objects = [
            (bucket['key'], bucket['doc_count'])
            for bucket in results_object
        ]

    return objects, total

def get_es_history(
    fields, filters=None,
    interval=DEFAULT_INTERVAL, aggregate_func=Aggregate.avg, sum_divide=False
):
    fields = [fields] if isinstance(fields, basestring) else fields

    q = get_es_query(index=settings.HISTORY_INDEXES)
    q.size(0)

    if filters:
        q.filter(Filter.and_(*filters))

    aggregates = []

    for field in fields:
        aggregates.append(aggregate_func(field, field))

    # Put the aggregates inside a date_histogram
    q.aggregate(
        Aggregate.date_histogram(
            'times', 'datetime', '{0}m'.format(interval)
        ).aggregate(*aggregates)
    )

    results = q.get()

    # Data is collected a 5min intervals, when we're summing we must divide by that interval
    interval_multipliter = 1
    if sum_divide:
        interval_multipliter = interval / 5

    date_histogram = []
    for bucket in results['aggregations']['times']['buckets']:
        date_bucket = {
            'datetime': bucket['key_as_string']
        }

        for field in fields:
            date_bucket[field] = bucket[field]['value'] / interval_multipliter

        date_histogram.append(date_bucket)

    return date_histogram
<commit_msg>Round output of history when dividing by interval.<commit_after># Source Server Stats
# File: sourcestats/util/elastic.py
# Desc: elasticsearch utilities

from datetime import datetime, timedelta

from flask import request, g
from elasticsearch import Elasticsearch
from elasticquery import ElasticQuery, Aggregate, Filter

from sourcestats import settings

from .request import in_request_args

ES_CLIENT = None
DEFAULT_INTERVAL = settings.COLLECT_INTERVAL / 60
SINCE_INTERVALS = {
    'd': 'days',
    'h': 'hours'
}

def get_es_client():
    global ES_CLIENT

    if ES_CLIENT is None:
        ES_CLIENT = Elasticsearch(settings.ES_HOSTS)

    return ES_CLIENT

def get_es_query(index=settings.SERVERS_INDEX):
    return ElasticQuery(
        es=get_es_client(),
        index=index,
        doc_type='server'
    )

def _get_since():
    if hasattr(g, 'request_since'):
        return g.request_since

    since = None

    # Attempt to parse any since
    if in_request_args('since'):
        interval = request.args['since'][-1]

        if interval in SINCE_INTERVALS:
            try:
                value = int(request.args['since'][0:-1])
                since = datetime.utcnow() - timedelta(**{
                    SINCE_INTERVALS[interval]: value
                })
            except ValueError:
                pass

    # Default to a day
    if since is None:
        since = datetime.utcnow() - timedelta(days=1)

    # ES mapping format has no microsecond
    since = since.replace(microsecond=0)

    # Cache for current request & return
    g.request_since = since
    return since

def _past_time(**kwargs):
    return (datetime.utcnow() - timedelta(**kwargs)).replace(microsecond=0)

def get_request_interval():
    since = _get_since()

    # If we're under a day, every 5 mins, like the collector
    if since >= _past_time(days=1):
        return DEFAULT_INTERVAL

    # If we're under one week, every 30m
    elif since >= _past_time(days=7):
        return 30

    # If we're under one month, every 2 hours
    elif since >= _past_time(days=31):
        return 120

    # Default a day
    return 1440

def get_request_filters():
    filters = []

    since = _get_since()
    filters.append(Filter.range('datetime', gte=since))

    for field in [
        'game_id',
        'map'
    ]:
        if in_request_args(field):
            filters.append(Filter.term(field, request.args[field]))

    return filters

def get_es_terms(field_name, filters=None, size=None, index=settings.SERVERS_INDEX):
    '''List 'objects', ie distinct field values from the indexes.'''
    q = get_es_query(index=index)
    q.size(0)

    if size is None:
        size = settings.ES_TERMS

    if filters:
        q.filter(Filter.and_(*filters))

    # Aggregates to generate terms and distinct values
    aggregates = (
        Aggregate.terms('objects', field_name, size=size),
        Aggregate.cardinality('values', field_name, precision_threshold=10000)
    )

    nested = False

    # Nested aggregate
    if '.' in field_name:
        nested_field, field_name = field_name.split('.')
        nested_aggregate = Aggregate.nested('nested', nested_field).aggregate(*aggregates)
        q.aggregate(nested_aggregate)
        nested = True

    # Normal aggregate
    else:
        q.aggregate(*aggregates)

    results = q.get()
    aggregations = results['aggregations']

    if nested:
        results_object = aggregations['nested']['objects']['buckets']
        total = aggregations['nested']['values']['value']
    else:
        results_object = aggregations['objects']['buckets']
        total = aggregations['values']['value']

    # For history docs the doc_counts in terms are useless as they'll just be duplicates
    if index == settings.HISTORY_INDEXES:
        objects = [bucket['key'] for bucket in results_object]

    # But in servers they represent what we see live
    else:
        objects = [
            (bucket['key'], bucket['doc_count'])
            for bucket in results_object
        ]

    return objects, total

def get_es_history(
    fields, filters=None,
    interval=DEFAULT_INTERVAL, aggregate_func=Aggregate.avg, sum_divide=False
):
    fields = [fields] if isinstance(fields, basestring) else fields

    q = get_es_query(index=settings.HISTORY_INDEXES)
    q.size(0)

    if filters:
        q.filter(Filter.and_(*filters))

    aggregates = []

    for field in fields:
        aggregates.append(aggregate_func(field, field))

    # Put the aggregates inside a date_histogram
    q.aggregate(
        Aggregate.date_histogram(
            'times', 'datetime', '{0}m'.format(interval)
        ).aggregate(*aggregates)
    )

    results = q.get()

    # Data is collected a 5min intervals, when we're summing we must divide by that interval
    interval_multipliter = 1
    if sum_divide:
        interval_multipliter = interval / 5

    date_histogram = []
    for bucket in results['aggregations']['times']['buckets']:
        date_bucket = {
            'datetime': bucket['key_as_string']
        }

        for field in fields:
            date_bucket[field] = round(bucket[field]['value'] / interval_multipliter)

        date_histogram.append(date_bucket)

    return date_histogram
<|endoftext|>


<commit_before>from __future__ import print_function
import os
import subprocess
import sys
import time

import click

from valohai_cli.ctx import get_project
from valohai_cli.git import expand_commit_id
from valohai_cli.messages import success
from valohai_cli.tui import get_spinner_character
from valohai_cli.utils import ensure_makedirs, format_size, sanitize_filename

MINIMUM_VALOHAI_LOCAL_RUN_VERSION = '0.2.1'

def print_parcel_progress(text):
    click.secho('::: %s' % text, bold=True, fg='cyan')

@click.command()
@click.option('-d', '--destination', type=click.Path(file_okay=False, dir_okay=True), required=False)
@click.option('-c', '--commit', help='Commit to read required Docker image IDs from')
@click.option(
    '--code',
    type=click.Choice(['bundle', 'archive', 'tarball', 'none']),
    default='bundle',
    help='Package code as a full Git bundle, an archive of Git HEAD, a tarball of the directory or not at all?',
)
@click.option('--docker-images/--no-docker-images', default=True, help='Package Docker images?')
@click.option('--valohai-local-run/--no-valohai-local-run', default=True, help='Download valohai-local-run + deps?')
def parcel(destination, commit, code, valohai_local_run, docker_images):
    project = get_project(require=True)

    if not destination:
        destination = '{}-parcel-{}'.format(
            project.name,
            time.strftime('%Y%m%d-%H%M%S'),
        )

    click.echo('Packing {} to directory {}'.format(
        click.style(project.name, bold=True, fg='blue'),
        click.style(destination, bold=True, fg='green'),
    ))

    ensure_makedirs(destination)

    extra_docker_images = []

    if code in ('bundle', 'archive', 'tarball'):
        export_code(project, destination, mode=code)

    if valohai_local_run:
        export_valohai_local_run(project, destination)

    if docker_images:
        export_docker_images(project, destination, commit, extra_docker_images)

    success('Parcel {} created!'.format(destination))

def export_valohai_local_run(project, destination):
    print_parcel_progress('Downloading valohai-local-run and dependencies')
    destination = os.path.join(destination, 'python-archives')
    subprocess.check_call(
        [
            'pip',
            '--disable-pip-version-check',
            'download',
            '--dest', destination,
            'valohai-local-run>=' + MINIMUM_VALOHAI_LOCAL_RUN_VERSION,
        ],
    )

def export_code(project, destination, mode):
    if mode == 'bundle':
        print_parcel_progress('Creating Git repository bundle')
        subprocess.check_call(
            ['git', 'bundle', 'create', os.path.join(destination, 'git-repo.bundle'), '--all'],
            cwd=project.directory,
        )
    elif mode == 'archive':
        print_parcel_progress('Creating Git single-revision archive')
        subprocess.check_call(
            ['git', 'archive', '--format', 'tar', 'HEAD', '-o', os.path.join(destination, 'git-commit.tar')],
            cwd=project.directory,
        )
    elif mode == 'tarball':
        print_parcel_progress('Creating full directory tarball')
        subprocess.check_call(
            [
                'tar',
                'cvf',
                os.path.join(destination, 'code.tar'),
                '--exclude=.git',
                '--exclude=%s' % os.path.basename(destination),  # In case we're being run in the same dir
                '.',  # Correct directory taken care of with cwd
            ],
            cwd=project.directory,
        )
    else:  # pragma: no cover
        raise NotImplementedError('...')

def get_docker_image_size(image):
    """
    Try to get the size of the given Docker image in bytes.
    :param image: The image name.
    :return: Number of bytes, or None if not available.
    :rtype: int|None
    """
    try:
        size = subprocess.check_output(
            ['docker', 'images', "--format={{.Size}}", image],
        ).strip().decode()
    except subprocess.CalledProcessError:  # Something went wrong, doesn't matter what.
        return None
    # No matter what, the Docker CLI API always outputs sizes
    # in a human-readable form, so we'll try and parse that...
    for suffix, multiplier in (('GB', 1e9), ('MB', 1e6), ('kB', 1e3), ('B', 1)):
        if size.endswith(suffix):
            return int(float(size[:-len(suffix)]) * multiplier)
    return None

def export_docker_images(project, destination, commit, extra_docker_images=()):
    commit = expand_commit_id(project.directory, commit=(commit or 'HEAD'))
    docker_images = set(step.image for step in project.get_config(commit).steps.values())
    docker_images |= set(extra_docker_images)
    for i, image in enumerate(docker_images, 1):
        output_path = os.path.join(destination, sanitize_filename('docker-{}.tar'.format(image)))
        if image not in extra_docker_images:
            print_parcel_progress('::: Pulling image {}/{} ({})'.format(
                i,
                len(docker_images),
                image,
            ))
            subprocess.check_call(['docker', 'pull', image])
        print_parcel_progress('::: Exporting image {}/{} ({})'.format(
            i,
            len(docker_images),
            image,
        ))
        export_docker_image(image, output_path)

def export_docker_image(image, output_path):
    """
    Export the Docker image `image` to the tar file `output_path`,
    with visual progress.

    :param image: Image specifier
    :param output_path: Output pathname
    """
    image_size = get_docker_image_size(image)
    proc = subprocess.Popen(['docker', 'save', image], bufsize=-1, stdout=subprocess.PIPE)
    with open(output_path, 'wb') as outfp:
        if sys.stdout.isatty():
            print('Initializing export...', end='\r')
        while proc.poll() is None:
            chunk = proc.stdout.read(1048576)
            if not chunk:
                break
            outfp.write(chunk)
            if sys.stdout.isatty():
                width = click.get_terminal_size()[0]
                status_text = '{} {}: {} / {}'.format(
                    get_spinner_character(),
                    image,
                    format_size(outfp.tell()),
                    (format_size(image_size) if image_size else 'unknown size'),
                )
                print(status_text.ljust(width - 1), end='\r')
    if proc.returncode:
        raise subprocess.CalledProcessError(proc.returncode, 'docker save ' + image)
    click.secho('    {} exported: {}'.format(
        image,
        format_size(os.stat(output_path).st_size),
    ), fg='green')
<commit_msg>parcel: add support for writing an unparcel script<commit_after>from __future__ import print_function
import os
import subprocess
import sys
import time

import click

from valohai_cli.ctx import get_project
from valohai_cli.git import expand_commit_id
from valohai_cli.messages import success
from valohai_cli.tui import get_spinner_character
from valohai_cli.utils import ensure_makedirs, format_size, sanitize_filename

MINIMUM_VALOHAI_LOCAL_RUN_VERSION = '0.2.1'

UNPARCEL_SCRIPT = r"""
#!/bin/bash
WORKING_COPY_DIR=$(mktemp -dut parcel-working-copy-XXXXXX)
(
set -ex
find . -name 'docker*tar' -exec docker load -i '{}' ';'
[ -d python-archives ] && pip3 install --user python-archives/*
[ -f git-repo.bundle ] && git clone ./git-repo.bundle "$WORKING_COPY_DIR"
[ -f git-commit.tar ] && mkdir "$WORKING_COPY_DIR" && tar -xvf ./git-commit.tar -C "$WORKING_COPY_DIR"
[ -f code.tar ] && mkdir "$WORKING_COPY_DIR" && tar -xvf ./code.tar -C "$WORKING_COPY_DIR"
)
cd $WORKING_COPY_DIR
""".strip()

def print_parcel_progress(text):
    click.secho('::: %s' % text, bold=True, fg='cyan')

@click.command()
@click.option('-d', '--destination', type=click.Path(file_okay=False, dir_okay=True), required=False)
@click.option('-c', '--commit', help='Commit to read required Docker image IDs from')
@click.option(
    '--code',
    type=click.Choice(['bundle', 'archive', 'tarball', 'none']),
    default='bundle',
    help='Package code as a full Git bundle, an archive of Git HEAD, a tarball of the directory or not at all?',
)
@click.option('--docker-images/--no-docker-images', default=True, help='Package Docker images?')
@click.option('--valohai-local-run/--no-valohai-local-run', default=True, help='Download valohai-local-run + deps?')
@click.option('--unparcel-script/--no-unparcel-script', default=True, help='Add unparcel script?')
def parcel(destination, commit, code, valohai_local_run, docker_images, unparcel_script):
    project = get_project(require=True)

    if not destination:
        destination = '{}-parcel-{}'.format(
            project.name,
            time.strftime('%Y%m%d-%H%M%S'),
        )

    click.echo('Packing {} to directory {}'.format(
        click.style(project.name, bold=True, fg='blue'),
        click.style(destination, bold=True, fg='green'),
    ))

    ensure_makedirs(destination)

    extra_docker_images = []

    if code in ('bundle', 'archive', 'tarball'):
        export_code(project, destination, mode=code)

    if valohai_local_run:
        export_valohai_local_run(project, destination)

    if docker_images:
        export_docker_images(project, destination, commit, extra_docker_images)

    if unparcel_script:
        write_unparcel_script(destination)

    success('Parcel {} created!'.format(destination))

def export_valohai_local_run(project, destination):
    print_parcel_progress('Downloading valohai-local-run and dependencies')
    destination = os.path.join(destination, 'python-archives')
    subprocess.check_call(
        [
            'pip',
            '--disable-pip-version-check',
            'download',
            '--dest', destination,
            'valohai-local-run>=' + MINIMUM_VALOHAI_LOCAL_RUN_VERSION,
        ],
    )

def write_unparcel_script(destination):
    unparcel_sh_path = os.path.join(destination, 'unparcel.sh')
    print_parcel_progress('Creating unparcel script {}'.format(unparcel_sh_path))
    with open(unparcel_sh_path, 'w') as outf:
        outf.write(UNPARCEL_SCRIPT)
    os.chmod(unparcel_sh_path, os.stat(unparcel_sh_path).st_mode | 0o700)

def export_code(project, destination, mode):
    if mode == 'bundle':
        print_parcel_progress('Creating Git repository bundle')
        subprocess.check_call(
            ['git', 'bundle', 'create', os.path.join(destination, 'git-repo.bundle'), '--all'],
            cwd=project.directory,
        )
    elif mode == 'archive':
        print_parcel_progress('Creating Git single-revision archive')
        subprocess.check_call(
            ['git', 'archive', '--format', 'tar', 'HEAD', '-o', os.path.join(destination, 'git-commit.tar')],
            cwd=project.directory,
        )
    elif mode == 'tarball':
        print_parcel_progress('Creating full directory tarball')
        subprocess.check_call(
            [
                'tar',
                'cvf',
                os.path.join(destination, 'code.tar'),
                '--exclude=.git',
                '--exclude=%s' % os.path.basename(destination),  # In case we're being run in the same dir
                '.',  # Correct directory taken care of with cwd
            ],
            cwd=project.directory,
        )
    else:  # pragma: no cover
        raise NotImplementedError('...')

def get_docker_image_size(image):
    """
    Try to get the size of the given Docker image in bytes.
    :param image: The image name.
    :return: Number of bytes, or None if not available.
    :rtype: int|None
    """
    try:
        size = subprocess.check_output(
            ['docker', 'images', "--format={{.Size}}", image],
        ).strip().decode()
    except subprocess.CalledProcessError:  # Something went wrong, doesn't matter what.
        return None
    # No matter what, the Docker CLI API always outputs sizes
    # in a human-readable form, so we'll try and parse that...
    for suffix, multiplier in (('GB', 1e9), ('MB', 1e6), ('kB', 1e3), ('B', 1)):
        if size.endswith(suffix):
            return int(float(size[:-len(suffix)]) * multiplier)
    return None

def export_docker_images(project, destination, commit, extra_docker_images=()):
    commit = expand_commit_id(project.directory, commit=(commit or 'HEAD'))
    docker_images = set(step.image for step in project.get_config(commit).steps.values())
    docker_images |= set(extra_docker_images)
    for i, image in enumerate(docker_images, 1):
        output_path = os.path.join(destination, sanitize_filename('docker-{}.tar'.format(image)))
        if image not in extra_docker_images:
            print_parcel_progress('::: Pulling image {}/{} ({})'.format(
                i,
                len(docker_images),
                image,
            ))
            subprocess.check_call(['docker', 'pull', image])
        print_parcel_progress('::: Exporting image {}/{} ({})'.format(
            i,
            len(docker_images),
            image,
        ))
        export_docker_image(image, output_path)

def export_docker_image(image, output_path):
    """
    Export the Docker image `image` to the tar file `output_path`,
    with visual progress.

    :param image: Image specifier
    :param output_path: Output pathname
    """
    image_size = get_docker_image_size(image)
    proc = subprocess.Popen(['docker', 'save', image], bufsize=-1, stdout=subprocess.PIPE)
    with open(output_path, 'wb') as outfp:
        if sys.stdout.isatty():
            print('Initializing export...', end='\r')
        while proc.poll() is None:
            chunk = proc.stdout.read(1048576)
            if not chunk:
                break
            outfp.write(chunk)
            if sys.stdout.isatty():
                width = click.get_terminal_size()[0]
                status_text = '{} {}: {} / {}'.format(
                    get_spinner_character(),
                    image,
                    format_size(outfp.tell()),
                    (format_size(image_size) if image_size else 'unknown size'),
                )
                print(status_text.ljust(width - 1), end='\r')
    if proc.returncode:
        raise subprocess.CalledProcessError(proc.returncode, 'docker save ' + image)
    click.secho('    {} exported: {}'.format(
        image,
        format_size(os.stat(output_path).st_size),
    ), fg='green')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from urllib.parse import urlparse
from collections import namedtuple
from concurrent.futures import ProcessPoolExecutor, as_completed

import requests

from lxml import html

from utils import remove_non_ascii_chars

# base url for eliteprospects.com
BASE_URL = "http://www.eliteprospects.com"
# url template for draft overview pages at eliteprospects.com
DRAFT_URL_TEMPLATE = "draft.php?year=%d"
# maximum worker count
MAX_WORKERS = 8
# named tuple to contain basic player information
Player = namedtuple(
    'Player', 'first_name last_name date_of_birth alt_last_name')

def retrieve_drafted_players_with_dobs(draft_year):
    """
    Retrieves basic player data (first name, last name, date of birth,
    alternate last name) from all player pages in the specified list.
    """
    # retrieving links to pages of all drafted players first
    player_urls = retrieve_drafted_player_links(draft_year)
    # setting up target list
    players_with_dobs = list()

    with ProcessPoolExecutor(max_workers=MAX_WORKERS) as processes:
        future_tasks = {
            processes.submit(
                get_player_with_dob, url): url for url in player_urls[:]}
        for future in as_completed(future_tasks):
            try:
                # TODO: think of something to do with the result here
                result = future.result()
                if result is not None:
                    players_with_dobs.append(result)
            except Exception as e:
                print
                print("Conccurrent task generated an exception: %s" % e)

    return players_with_dobs

def get_player_with_dob(url):
    """
    Retrieves single player along with date of birth.
    """
    req = requests.get(url)
    print("+ Retrieving player information from %s" % url)
    doc = html.fromstring(req.text)

    # retrieving birthdate url that contains all necessary information in
    # granular form, i.e. 1998-04-19
    dob_url = doc.xpath("//a[starts-with(@href, 'birthdate')]/@href")
    if not dob_url:
        return
    dob_url = dob_url.pop(0)
    # retrieving player information from retrieved url
    dob, first_name, last_name = get_player_details_from_url(dob_url)

    # retrieving alternate last name (if applicable)
    alt_last_name = ''
    aka_element = doc.xpath("//font[starts-with(text(), 'a.k.a.')]/text()")
    if aka_element:
        aka = aka_element.pop(0).replace("a.k.a.", "").replace('"', "").strip()
        # retrieving all available alternate names
        akas = [a.strip() for a in aka.split(",")]
        tmp_alt_last_names = list()
        for aka in akas:
            # trying to remove already known first name
            aka_wo_first_name = aka.replace(first_name, "").strip()
            # splitting remaining alternate name
            tokens = aka_wo_first_name.split()
            # if known first name was actually removed, the rest is
            # the alternate last name
            if aka != aka_wo_first_name:
                tmp_alt_last_names.append(aka_wo_first_name)
            # if there's just one token after the split and first letters in
            # split result and original name match: this token is the alternate
            # last name
            elif len(tokens) == 1:
                if tokens[0][0].lower() == last_name.lower()[0]:
                    tmp_alt_last_names.append(tokens[0])
            # if there are two tokens after the splot: first one is alternate
            # first, second one alternate last name
            elif len(tokens) == 2:
                tmp_alt_last_names.append(tokens[-1])
            else:
                # TODO: logger warning
                print(
                    "Unable to retrieve alternate last " +
                    "name for %s %s from: %s" % (first_name, last_name, aka))
        print(tmp_alt_last_names)
        # using only first found alternate last name
        # TODO: use all alternate names
        if tmp_alt_last_names and tmp_alt_last_names[0] != last_name:
            alt_last_name = tmp_alt_last_names[0]

    # adding current player to list dictionary of players w/ date of births
    return Player(
        remove_non_ascii_chars(first_name),
        remove_non_ascii_chars(last_name),
        dob, alt_last_name)

def retrieve_drafted_player_links(draft_year):
    """
    Retrieves links to player pages for all players drafted in the specified
    draft year.
    """
    url = "/".join((BASE_URL, DRAFT_URL_TEMPLATE % draft_year))
    req = requests.get(url)
    doc = html.fromstring(req.text)

    print(
        "+ Retrieving urls to pages of each player drafted in %d" % draft_year)

    # stub links to player pages are present at the specified position in
    # the main table
    return ["/".join((BASE_URL, link)) for link in doc.xpath(
        "//tr[@bordercolor='#FFFFFF']/td[3]/a/@href")]

def get_player_details_from_url(dob_url):
    """
    Gets player details, i.e. first name, last name and date of birth, from
    specifield url.
    """
    # exploding url into its components
    url_comps = urlparse(dob_url)
    # retrieving player details by exploding each part of the url's
    # query component
    dob, first_name, last_name = [
        comp.split("=")[-1] for comp in url_comps.query.split("&")]
    return dob, first_name, last_name
<commit_msg>Adjust search for alternate last name<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from urllib.parse import urlparse
from collections import namedtuple
from concurrent.futures import ProcessPoolExecutor, as_completed

import requests

from lxml import html

from utils import remove_non_ascii_chars

# base url for eliteprospects.com
BASE_URL = "http://www.eliteprospects.com"
# url template for draft overview pages at eliteprospects.com
DRAFT_URL_TEMPLATE = "draft.php?year=%d"
# maximum worker count
MAX_WORKERS = 8
# named tuple to contain basic player information
Player = namedtuple(
    'Player', 'first_name last_name date_of_birth alt_last_name')

def retrieve_drafted_players_with_dobs(draft_year):
    """
    Retrieves basic player data (first name, last name, date of birth,
    alternate last name) from all player pages in the specified list.
    """
    # retrieving links to pages of all drafted players first
    player_urls = retrieve_drafted_player_links(draft_year)
    # setting up target list
    players_with_dobs = list()

    with ProcessPoolExecutor(max_workers=MAX_WORKERS) as processes:
        future_tasks = {
            processes.submit(
                get_player_with_dob, url): url for url in player_urls[:]}
        for future in as_completed(future_tasks):
            try:
                # TODO: think of something to do with the result here
                result = future.result()
                if result is not None:
                    players_with_dobs.append(result)
            except Exception as e:
                print
                print("Conccurrent task generated an exception: %s" % e)

    return players_with_dobs

def get_player_with_dob(url):
    """
    Retrieves single player along with date of birth.
    """
    req = requests.get(url)
    print("+ Retrieving player information from %s" % url)
    # NB: there's a problem with eliteprospects player pages: officially
    # encoded in iso-8859-1 they at times contain plain unicode characters,
    # e.g. for alternate names (see Dominik Lakatos/Lakatoš, url:
    # http://www.eliteprospects.com/player.php?player=195562)
    # this is obviously a data problem that couldn't have been solved within
    # the context of this application yet, manual post-processing of alternate
    # names may therefore be necessary
    doc = html.fromstring(req.text)

    # retrieving birthdate url that contains all necessary information in
    # granular form, i.e. 1998-04-19
    dob_url = doc.xpath("//a[starts-with(@href, 'birthdate')]/@href")
    if not dob_url:
        return
    dob_url = dob_url.pop(0)
    # retrieving player information from retrieved url
    dob, first_name, last_name = get_player_details_from_url(dob_url)

    # retrieving alternate last name (if applicable)
    alt_last_name = ''
    aka_element = doc.xpath("//font[starts-with(text(), 'a.k.a.')]/text()")
    if aka_element:
        aka = aka_element.pop(0).replace("a.k.a.", "").replace('"', "").strip()
        # retrieving all available alternate names
        akas = [a.strip() for a in aka.split(",")]
        tmp_alt_last_names = list()
        for aka in akas:
            # trying to remove already known first name
            aka_wo_first_name = aka.replace(first_name, "").strip()
            # splitting remaining alternate name
            tokens = aka_wo_first_name.split()
            # if known first name was actually removed, the rest is
            # the alternate last name
            # if we above just had a first name removed that was the short form
            # of a longer one, i.e. Nicklaus Perbix - Nick => laus Perbix, the
            # remaining part o/c does not represent an alternate last name
            if aka != aka_wo_first_name and not aka_wo_first_name.endswith(
                    last_name):
                        tmp_alt_last_names.append(aka_wo_first_name)
            # if there's just one token after the split and first letters in
            # split result and original name match: this token is the alternate
            # last name
            elif len(tokens) == 1:
                if tokens[0][0].lower() == last_name.lower()[0]:
                    tmp_alt_last_names.append(tokens[0])
            # if there are two tokens after the split: first one is alternate
            # first, second one alternate last name
            elif len(tokens) == 2:
                tmp_alt_last_names.append(tokens[-1])
            else:
                # TODO: logger warning
                print(
                    "Unable to retrieve alternate last " +
                    "name for %s %s from: %s" % (first_name, last_name, aka))
        # using only first found alternate last name
        # TODO: use all alternate names
        if tmp_alt_last_names and tmp_alt_last_names[0] != last_name:
            alt_last_name = tmp_alt_last_names[0]

    # adding current player to list dictionary of players w/ date of births
    return Player(
        remove_non_ascii_chars(first_name),
        remove_non_ascii_chars(last_name),
        dob, alt_last_name)

def retrieve_drafted_player_links(draft_year):
    """
    Retrieves links to player pages for all players drafted in the specified
    draft year.
    """
    url = "/".join((BASE_URL, DRAFT_URL_TEMPLATE % draft_year))
    req = requests.get(url)
    doc = html.fromstring(req.text)

    print(
        "+ Retrieving urls of Eliteprospects profiles " +
        "for each player drafted in %d" % draft_year)

    # stub links to player pages are present at the specified position in
    # the main table
    return ["/".join((BASE_URL, link)) for link in doc.xpath(
        "//tr[@bordercolor='#FFFFFF']/td[3]/a/@href")]

def get_player_details_from_url(dob_url):
    """
    Gets player details, i.e. first name, last name and date of birth, from
    specifield url.
    """
    # exploding url into its components
    url_comps = urlparse(dob_url)
    # retrieving player details by exploding each part of the url's
    # query component
    dob, first_name, last_name = [
        comp.split("=")[-1] for comp in url_comps.query.split("&")]
    return dob, first_name, last_name
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
Script that uploads files in entered directory as an archive to transfer.sh, for easy downloading.
"""

import os
import sys
import zipfile
import requests
import datetime
import pyperclip

def create_zip():
    if len(sys.argv) > 1:
        path_to_folder = sys.argv[1]
    else:
        path_to_folder = input("Please enter directory: \n")

    if os.path.isdir(path_to_folder):
        os.chdir(path_to_folder)
        zip_name = 'files_archive_{}.zip'.format(str(datetime.datetime.now())[5:16].replace(' ', "_"))
        files = os.listdir()
        print("Creating zipfile from files in...", path_to_folder)
        with zipfile.ZipFile(zip_name, 'w') as zip:
            for img in files:
                zip.write(img)
                print("Added file: ", img)

        return zip_name
    else:
        print("Please enter a valid absolute path to directory")
        sys.exit()

def send_zip():
    zip_file = create_zip()
    size_of_file = get_archive_size(zip_file)
    final_date = get_date_in_two_weeks()

    print("\nSending zipfile: {} (size of the file: {} MB)".format(zip_file, size_of_file))
    url = 'https://transfer.sh/'
    file = {'file': open(zip_file, 'rb')}
    response = requests.post(url, files=file)
    download_link = response.content.decode('utf-8')
    print("Link to download zipfile(will be saved till {}):\n{}".format(final_date, download_link))

    try:
        pyperclip.copy(download_link)
    except:
        print("There is no copy/paste environment, please install one of the following packages:\n"
              "sudo apt-get install\n"
              "sudo apt-get install xclip")
    print("Link copied to clipboard")
    confirm_removal(zip_file)

def get_date_in_two_weeks():
    today = datetime.datetime.today()
    date_in_two_weeks = today + datetime.timedelta(days=14)
    return date_in_two_weeks.date()

def get_archive_size(file):
    size_in_bytes = os.path.getsize(file)
    size_in_megabytes = size_in_bytes / 1000000
    return size_in_megabytes

def confirm_removal(zip_file):
    confirmation = input('\nDelete files in the directory?(y/n, Y/N, archive): ')
    if (confirmation == 'y') or (confirmation == 'Y'):
        clean_up()
    elif (confirmation == 'n') or (confirmation == 'N'):
        print("OK, files will be there")
        sys.exit()
    elif confirmation == 'archive':
        remove_archive(zip_file)
    else:
        print("Please enter a valid answer (y/n, Y/N)")
        confirm_removal()

def remove_archive(file_name):
    print("Deleting archive: ", file_name)
    os.remove(file_name)
    print("File removed")

def clean_up():
    print('\nCleaning up...')
    files = os.listdir()
    for f in files:
        os.remove(f)
        print("Removed file: ", f)

def print_help():
    print("Usage:\n transfer_files /path/to/files - archive directly"
          "\n transfer_files - start script with prompt "
          "\n transfer_files -h - print help message")

def main():
    if len(sys.argv) > 1 and sys.argv[1] == '-h':
        print_help()
    else:
        try:
            send_zip()
        except KeyboardInterrupt:
            print("\nExecution stopped")

if __name__ == '__main__':
    main()
<commit_msg>Updated app, added arguments and interactive mode of running<commit_after>#!/usr/bin/env python3

"""
Script that uploads files in entered directory as an archive to transfer.sh, for easy downloading.
"""

import os
import sys
import zipfile
import requests
import datetime
import pyperclip
import optparse

# TODO: overall refactor (namely send_file and send_zip)

def get_date_in_two_weeks():
    today = datetime.datetime.today()
    date_in_two_weeks = today + datetime.timedelta(days=14)
    return date_in_two_weeks.date()

def get_size(file):
    size_in_bytes = os.path.getsize(file)
    size_in_megabytes = size_in_bytes / 1000000
    return size_in_megabytes

def parse_params():
    usage = "%prog [-i/--file/--directory] /path/to/file [--rm-archive/--rm-file]\n" \
            "Short description:\n" \
            "transfersh-client is used to send files to transfer.sh and retrieve download link, " \
            "so it could be shared fast and easily directly from command-line.\n" \
            "It can send all files from entered directory (in an archive) or send one file.\n" \
            "You can download up to 10 GB files to transfer.sh, they will be saved there for 14 days."
    parser = optparse.OptionParser(usage)
    parser.add_option('-i', '--interactive',
                      dest="interactive_mode",
                      action="store_true",
                      help="run in interactive mode (with entering info in the prompt)")
    parser.add_option('-d', '--directory',
                      dest="directory",
                      action="store",
                      type="string",
                      help="enter absolute path to directory, and create archive of it")
    parser.add_option('-f', '--file',
                      dest="file",
                      action="store",
                      type="string",
                      help="absolute path to file which will be uploaded")
    parser.add_option('--rf', '--rm-file',
                      dest="rm_file",
                      action="store_true",
                      help="remove files after sending")
    parser.add_option('--ra', '--rm-archive',
                      dest="rm_arch",
                      action="store_true",
                      help="remove only created archive")
    (options, args) = parser.parse_args()

    return options, args, parser

def check_params():
    params = parse_params()
    options = params[0]
    parser = params[2]

    # handle incorrect usage of options (a bit of spaghetti code)
    if (options.interactive_mode and options.file) \
            or (options.interactive_mode and options.directory) \
            or (options.interactive_mode and options.rm_arch) \
            or (options.interactive_mode and options.rm_file):
        print("Interactive option is used separately from other options\n")
        parser.print_help()
        sys.exit()
    elif (options.file and options.rm_arch) \
            or (options.directory and options.rm_file):
        print("Use either --ra or --rf accordingly:\n")
        parser.print_help()
        sys.exit()
    elif (options.file and options.directory) \
            or (options.rm_arch and options.rm_file):
        print("Use either -f or -d options\n")
        parser.print_help()
        sys.exit()
    else:
        return options

def handle_params():
    options = check_params()

    # convert entered directory or file to its absolute path
    if options.file is not None and os.path.isabs(options.file) is False:
        current_dir = os.getcwd()
        if str(options.file).startswith("./"):
            options.file = current_dir + options.file[1:]
        else:
            options.file = current_dir + "/" + options.file

    if options.directory is not None and os.path.isabs(options.directory) is False:
        current_dir = os.getcwd()
        if str(options.directory).startswith("./"):
            options.directory = current_dir + options.directory[1:]
        else:
            options.directory = current_dir + "/" + options.directory

    if options.interactive_mode or len(sys.argv) == 1:
        interactive_mode_run()

    # check if file or directory exists, and weather to remove created archive or file
    if options.file is not None:
        try:
            os.path.isfile(options.file)
        except FileNotFoundError:
            print("Please enter valid absolute path to file")
            sys.exit()
        else:
            if options.rm_file:
                file_path = send_file(options.file)
                remove_file(file_path)
            else:
                send_file(options.file)

    if options.directory is not None:
        try:
            os.path.isdir(options.directory)
        except NotADirectoryError:
            print("Please enter valid absolute path to directory")
            sys.exit()
        else:
            if options.rm_arch:
                zipfile = send_zip(options.directory)
                remove_file(zipfile)
            else:
                send_zip(options.directory)

def interactive_mode_run():
    path = input("Enter path to file or directory:\n ")
    if os.path.isabs(path) is False:
        if path.startswith('./'):
            abs_path = os.getcwd() + path[1:]
        else:
            abs_path = os.getcwd() + "/" + path

    if os.path.isfile(abs_path):
        send_file(abs_path)
        confirm = input("Remove file? (y/n, yes/no):")
        confirm_removal(confirm, abs_path)
    elif os.path.isdir(abs_path):
        arch_name = send_zip(abs_path)
        confirm = input("Remove archive? (y/n, yes/no):")
        confirm_removal(confirm, arch_name)
    else:
        print("Please enter a valid absolute path to file/directory")
        sys.exit()

def create_zip(file_dir):
    os.chdir(file_dir)
    zip_name = 'files_archive_{}.zip'.format(str(datetime.datetime.now())[5:16].replace(' ', "_"))
    files = os.listdir()
    print("Creating zipfile from files in...", file_dir)
    with zipfile.ZipFile(zip_name, 'w') as zip:
        for img in files:
            zip.write(img)
            print("Added file: ", img)

    return zip_name

def remove_file(file):
    print("Removing file...", file)
    os.remove(file)
    print("Removed.")

def confirm_removal(confirm, filename):
    if confirm == 'y' or confirm == 'yes':
        remove_file(filename)
    elif confirm == 'n' or confirm == 'no':
        print("File will stay there")
    else:
        print("Please etner a valid answer (y/n, yes/no)")
        confirm_removal()

def send_zip(file_dir):
    zip_file = create_zip(file_dir)
    size_of_file = get_size(zip_file)
    final_date = get_date_in_two_weeks()

    print("\nSending zipfile: {} (size of the file: {} MB)".format(zip_file, size_of_file))
    url = 'https://transfer.sh/'
    file = {'{}'.format(zip_file): open(zip_file, 'rb')}
    response = requests.post(url, files=file)
    download_link = response.content.decode('utf-8')
    print("Link to download zipfile(will be saved till {}):\n{}".format(final_date, download_link))

    try:
        pyperclip.copy(download_link)
    except:
        print("There is no copy/paste environment, please install one of the following packages:\n"
              "sudo apt-get install\n"
              "sudo apt-get install xclip")
    print("Link copied to clipboard")

    # convert to absolute path (just for double check)
    abs_path_to_archive = os.getcwd() + "/" + zip_file
    return abs_path_to_archive

def send_file(file_path):
    size_of_file = get_size(file_path)
    final_date = get_date_in_two_weeks()
    filename_from_path = os.path.basename(file_path)

    print("\nSending file: {} (size of the file: {} MB)".format(file_path, size_of_file))
    url = 'https://transfer.sh/'
    file = {'{}'.format(filename_from_path): open(filename_from_path, 'rb')}
    response = requests.post(url, files=file)
    download_link = response.content.decode('utf-8')
    print("Link to download file(will be saved till {}):\n{}".format(final_date, download_link))

    try:
        pyperclip.copy(download_link)
    except:
        print("There is no copy/paste environment, please install one of the following packages:\n"
              "sudo apt-get install\n"
              "sudo apt-get install xclip")
    print("Link copied to clipboard")
    return file_path

def main():
    try:
        handle_params()
    except KeyboardInterrupt:
        print('Execution stopped')
        sys.exit()

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>import copy
import warnings

from LabelProperties import LabelProperties

class PlotInfo(object):
    def __init__(self,
                 plotType,
                 xValues = None,
                 yValues = None,
                 xTickLabels = None,
                 yTickLabels = None,
                 xTickLabelPoints = None,
                 yTickLabelPoints = None,
                 xTickLabelProperties = None,
                 yTickLabelProperties = None,
                 label = None,
                 yMins = None,
                 yMaxes = None,
                 yErrors = None,
                 autosort = True,
                 xLimits = None,
                 errorBarColor = None,
                 errorBarLineWidth = None,
                 errorBarCapsize = None,
                 picker=None):

        if xValues is None:
            xValues = []

        if yValues is None:
            yValues = []

        if xTickLabels is None:
            xTickLabels = []

        if yTickLabels is None:
            yTickLabels = []

        if xTickLabelPoints is None:
            xTickLabelPoints = []

        if yTickLabelPoints is None:
            yTickLabelPoints = []

        if yMins is None:
            yMins = []

        if yMaxes is None:
            yMaxes = []

        if yErrors is None:
            yErrors = []

        self.plotType = plotType
        """
        Names the type of plot element
        """

        self.xValues = xValues
        """
        An array of x-axis values for the plot element
        """

        self.yValues = yValues
        """
        An array of y-axis values for the plot elements
        """

        self.xTickLabels = xTickLabels
        """
        A list of labels that should be drawn on the x-axis
        """

        self.yTickLabels = yTickLabels
        """
        A list of labels that should be drawn on the y-axis
        """

        self.xTickLabelPoints = xTickLabelPoints
        """
        The locations on the x-axis where the labels in :attr:`xTickLabels`
        should be drawn
        """

        self.yTickLabelPoints = yTickLabelPoints
        """
        The locations on the x-axis where the labels in :attr:`xTickLabels`
        should be drawn
        """

        self._xTickLabelProperties = LabelProperties()

        self._yTickLabelProperties = LabelProperties()

        if xTickLabelProperties is not None:
            self.xTickLabelProperties = xTickLabelProperties

        if yTickLabelProperties is not None:
            self.yTickLabelProperties = yTickLabelProperties

        self.label = label
        """
        A string used to label this plot element in a legend
        """

        self.yMins = yMins
        """
        For asymmetric error bars, a list of locations for the bottoms of this
        plot element's error bars
        """

        self.yMaxes = yMaxes
        """
        For asymmetric error bars, a list of locations for the tops of this
        plot element's error bars
        """

        self.yErrors = yErrors
        """
        For symmetric error bars, a list of error bar widths for this plot
        element's error bars
        """

        self.autosort = autosort
        """
        If true, x/y pairs in :attrs:`xValues` and :attrs:`yValues` are sorted
        by x value before the plot is rendered
        """

        self.xLimits = xLimits

        self.errorBarColor = errorBarColor
        self.errorBarLineWidth = errorBarLineWidth
        self.errorBarCapsize = errorBarCapsize

        self.picker = picker

    @property
    def xTickLabelProperties(self):
        """
        A dictionary of properties that control the appearance of the X axis'
        tick labels. See :class:`boomslang.LabelProperties.LabelProperties` for
        more information on which properties can be set.
        """

        return self._xTickLabelProperties

    @xTickLabelProperties.setter
    def xTickLabelProperties(self, **props):
        self._xTickLabelProperties.update(props)

    @xTickLabelProperties.setter
    def xTickLabelProperties(self, propsobj):
        self._xTickLabelProperties.update(propsobj)

    @property
    def yTickLabelProperties(self):
        """
        A dictionary of properties that control the appearance of the Y axis'
        tick labels. See :class:`boomslang.LabelProperties.LabelProperties` for
        more information on which properties can be set.
        """

        return self._yTickLabelProperties

    @yTickLabelProperties.setter
    def yTickLabelProperties(self, props):
        self._yTickLabelProperties.update(props)

    def __str__(self):
        return str(self.__dict__)

    def _preDraw(self):
        """
        The _preDraw function is called on all plot elements before drawing
        occurs. This allows various fields to be set in a structured way
        prior to actual drawing.
        """
        pass

    def setXTickLabelProperties(self, **propList):
        # Deprecated: use xTickLabelProperties field instead
        self._xTickLabelProperties.update(propList)

    def setYTickLabelProperties(self, **propList):
        # Deprecated: use yTickLabelProperties field instead
        self._yTickLabelProperties.update(propList)

    def split(self, pieces):
        """
        Split this plot element into `pieces` separate plot elements, each of
        which is responsible for a disjoint range of the original plot
        element's x-axis. This is useful for situations where a plot element
        cannot fit comfortably in a single plot.
        """
        elements = []

        numXVals = len(self.xValues)

        valChunkSize = numXVals / pieces
        valChunkRemainder = numXVals % pieces

        for i in xrange(pieces):
            element = copy.deepcopy(self)

            if i < pieces - 1 or valChunkRemainder == 0:
                element.xValues = self.xValues[i * valChunkSize:(i+1) * valChunkSize]
                element.yValues = self.yValues[i * valChunkSize:(i+1) * valChunkSize]
            else:
                element.xValues = self.xValues[i * valChunkSize:]
                element.yValues = self.yValues[i * valChunkSize:]
            elements.append(element)
        return elements

    def getAttributes(self):
        """
        Get the attributes dictionary used to style the plot element. Extending
        this method allows plot elements to inherit attributes.
        """

        kwdict = {}

        if self.picker is not None:
            kwdict['picker'] = self.picker

        return kwdict

    def draw(self, fig, axis, transform=None):
        """
        The base method for drawing a plot element on the axis `axis` within
        the figure `fig`. Other plot elements should override this method, but
        should also be sure to call this method before doing any
        element-specific processing.
        """

        if len(self.xValues) > 0 and self.autosort:
            # This is a total kludge --AR

            sortAsymmetricErrorBars = len(self.yMins) > 0 or \
                len(self.yMaxes) > 0
            sortSymmetricErrorBars = len(self.yErrors) > 0

            if sortAsymmetricErrorBars:
                if len(self.yMins) != len(self.yValues):
                    warnings.warn("You don't have an error bar for every "
                                  "point, some points will be truncated")

                zipped = zip(self.xValues, self.yValues, self.yMins,
                             self.yMaxes)
                zipped.sort()
                self.xValues, self.yValues, self.yMins, self.yMaxes \
                    = zip(*zipped)
            elif sortSymmetricErrorBars:
                if len(self.yErrors) != len(self.yValues):
                    warnings.warn("You don't have an error bar for every "
                                  "point, some points will be truncated")

                zipped = zip(self.xValues, self.yValues, self.yErrors)
                zipped.sort()
                self.xValues, self.yValues, self.yErrors = zip(*zipped)
            else:
                zipped = zip(self.xValues, self.yValues)
                zipped.sort()
                self.xValues, self.yValues = zip(*zipped)

        if len(self.xTickLabels) > 0:
            if len(self.xTickLabelPoints) == 0:
                axis.set_xticks(self.xValues[0:len(self.xTickLabels)])
            else:
                axis.set_xticks(self.xTickLabelPoints)

            axis.set_xticklabels(self.xTickLabels, **self.xTickLabelProperties)

        if len(self.yTickLabels) > 0:
            if len(self.yTickLabelPoints) == 0:
                axis.set_yticks(self.yValues[0:len(self.yTickLabels)])
            else:
                axis.set_yticks(self.yTickLabelPoints)

            axis.set_yticklabels(self.yTickLabels, **self.yTickLabelProperties)

        self.drawErrorBars(axis)

    def drawErrorBars(self, axis, transform=None):
        """
        Draw error bars for the plot element using the plot element's
        `yMins`/`yMaxes` or `yErrors`. If `yMins` or `yMaxes` have non-zero
        length, they are used to draw error bars that can be
        asymmetric. Otherwise, `yErrors` are used to draw symmetric error bars.

        If some transform should be applied to the error bars, it is provided
        by the caller with the `transform` parameter.
        """

        errorBarKeywords = {}
        if  self.errorBarColor is not None:
            errorBarKeywords["ecolor"] = self.errorBarColor
        elif hasattr(self, "color"):
            errorBarKeywords["ecolor"] = self.color

        if self.errorBarCapsize is not None:
            errorBarKeywords["capsize"] = self.errorBarCapsize

        if self.errorBarLineWidth is not None:
            errorBarKeywords["linewidth"] = self.errorBarLineWidth

        if hasattr(self, "lineStyle"):
            errorBarKeywords["linestyle"] = self.lineStyle

        if transform:
            errorBarKeywords['transform'] = transform + axis.transData

        errorBarKeywords["fmt"] = None

        if len(self.yMins) > 0 and len(self.yMaxes) > 0:
            numYVals = len(self.yValues)
            yMin = [self.yValues[i] - self.yMins[i] for i in xrange(numYVals)]
            yMax = [self.yMaxes[i] - self.yValues[i] for i in xrange(numYVals)]

            errorBarKeywords["yerr"] = [yMin, yMax]

            axis.errorbar(self.xValues, self.yValues, **errorBarKeywords)
        elif len(self.yErrors) > 0:
            errorBarKeywords["yerr"] = self.yErrors
            axis.errorbar(self.xValues, self.yValues, **errorBarKeywords)

<commit_msg>Fixing documentation typo.<commit_after>import copy
import warnings

from LabelProperties import LabelProperties

class PlotInfo(object):
    def __init__(self,
                 plotType,
                 xValues = None,
                 yValues = None,
                 xTickLabels = None,
                 yTickLabels = None,
                 xTickLabelPoints = None,
                 yTickLabelPoints = None,
                 xTickLabelProperties = None,
                 yTickLabelProperties = None,
                 label = None,
                 yMins = None,
                 yMaxes = None,
                 yErrors = None,
                 autosort = True,
                 xLimits = None,
                 errorBarColor = None,
                 errorBarLineWidth = None,
                 errorBarCapsize = None,
                 picker=None):

        if xValues is None:
            xValues = []

        if yValues is None:
            yValues = []

        if xTickLabels is None:
            xTickLabels = []

        if yTickLabels is None:
            yTickLabels = []

        if xTickLabelPoints is None:
            xTickLabelPoints = []

        if yTickLabelPoints is None:
            yTickLabelPoints = []

        if yMins is None:
            yMins = []

        if yMaxes is None:
            yMaxes = []

        if yErrors is None:
            yErrors = []

        self.plotType = plotType
        """
        Names the type of plot element
        """

        self.xValues = xValues
        """
        An array of x-axis values for the plot element
        """

        self.yValues = yValues
        """
        An array of y-axis values for the plot elements
        """

        self.xTickLabels = xTickLabels
        """
        A list of labels that should be drawn on the x-axis
        """

        self.yTickLabels = yTickLabels
        """
        A list of labels that should be drawn on the y-axis
        """

        self.xTickLabelPoints = xTickLabelPoints
        """
        The locations on the x-axis where the labels in :attr:`xTickLabels`
        should be drawn
        """

        self.yTickLabelPoints = yTickLabelPoints
        """
        The locations on the x-axis where the labels in :attr:`xTickLabels`
        should be drawn
        """

        self._xTickLabelProperties = LabelProperties()

        self._yTickLabelProperties = LabelProperties()

        if xTickLabelProperties is not None:
            self.xTickLabelProperties = xTickLabelProperties

        if yTickLabelProperties is not None:
            self.yTickLabelProperties = yTickLabelProperties

        self.label = label
        """
        A string used to label this plot element in a legend
        """

        self.yMins = yMins
        """
        For asymmetric error bars, a list of locations for the bottoms of this
        plot element's error bars
        """

        self.yMaxes = yMaxes
        """
        For asymmetric error bars, a list of locations for the tops of this
        plot element's error bars
        """

        self.yErrors = yErrors
        """
        For symmetric error bars, a list of error bar widths for this plot
        element's error bars
        """

        self.autosort = autosort
        """
        If true, x/y pairs in :py:attr:`xValues` and :py:attr:`yValues` are
        sorted by x value before the plot is rendered
        """

        self.xLimits = xLimits

        self.errorBarColor = errorBarColor
        self.errorBarLineWidth = errorBarLineWidth
        self.errorBarCapsize = errorBarCapsize

        self.picker = picker

    @property
    def xTickLabelProperties(self):
        """
        A dictionary of properties that control the appearance of the X axis'
        tick labels. See :class:`boomslang.LabelProperties.LabelProperties` for
        more information on which properties can be set.
        """

        return self._xTickLabelProperties

    @xTickLabelProperties.setter
    def xTickLabelProperties(self, **props):
        self._xTickLabelProperties.update(props)

    @xTickLabelProperties.setter
    def xTickLabelProperties(self, propsobj):
        self._xTickLabelProperties.update(propsobj)

    @property
    def yTickLabelProperties(self):
        """
        A dictionary of properties that control the appearance of the Y axis'
        tick labels. See :class:`boomslang.LabelProperties.LabelProperties` for
        more information on which properties can be set.
        """

        return self._yTickLabelProperties

    @yTickLabelProperties.setter
    def yTickLabelProperties(self, props):
        self._yTickLabelProperties.update(props)

    def __str__(self):
        return str(self.__dict__)

    def _preDraw(self):
        """
        The _preDraw function is called on all plot elements before drawing
        occurs. This allows various fields to be set in a structured way
        prior to actual drawing.
        """
        pass

    def setXTickLabelProperties(self, **propList):
        # Deprecated: use xTickLabelProperties field instead
        self._xTickLabelProperties.update(propList)

    def setYTickLabelProperties(self, **propList):
        # Deprecated: use yTickLabelProperties field instead
        self._yTickLabelProperties.update(propList)

    def split(self, pieces):
        """
        Split this plot element into `pieces` separate plot elements, each of
        which is responsible for a disjoint range of the original plot
        element's x-axis. This is useful for situations where a plot element
        cannot fit comfortably in a single plot.
        """
        elements = []

        numXVals = len(self.xValues)

        valChunkSize = numXVals / pieces
        valChunkRemainder = numXVals % pieces

        for i in xrange(pieces):
            element = copy.deepcopy(self)

            if i < pieces - 1 or valChunkRemainder == 0:
                element.xValues = self.xValues[i * valChunkSize:(i+1) * valChunkSize]
                element.yValues = self.yValues[i * valChunkSize:(i+1) * valChunkSize]
            else:
                element.xValues = self.xValues[i * valChunkSize:]
                element.yValues = self.yValues[i * valChunkSize:]
            elements.append(element)
        return elements

    def getAttributes(self):
        """
        Get the attributes dictionary used to style the plot element. Extending
        this method allows plot elements to inherit attributes.
        """

        kwdict = {}

        if self.picker is not None:
            kwdict['picker'] = self.picker

        return kwdict

    def draw(self, fig, axis, transform=None):
        """
        The base method for drawing a plot element on the axis `axis` within
        the figure `fig`. Other plot elements should override this method, but
        should also be sure to call this method before doing any
        element-specific processing.
        """

        if len(self.xValues) > 0 and self.autosort:
            # This is a total kludge --AR

            sortAsymmetricErrorBars = len(self.yMins) > 0 or \
                len(self.yMaxes) > 0
            sortSymmetricErrorBars = len(self.yErrors) > 0

            if sortAsymmetricErrorBars:
                if len(self.yMins) != len(self.yValues):
                    warnings.warn("You don't have an error bar for every "
                                  "point, some points will be truncated")

                zipped = zip(self.xValues, self.yValues, self.yMins,
                             self.yMaxes)
                zipped.sort()
                self.xValues, self.yValues, self.yMins, self.yMaxes \
                    = zip(*zipped)
            elif sortSymmetricErrorBars:
                if len(self.yErrors) != len(self.yValues):
                    warnings.warn("You don't have an error bar for every "
                                  "point, some points will be truncated")

                zipped = zip(self.xValues, self.yValues, self.yErrors)
                zipped.sort()
                self.xValues, self.yValues, self.yErrors = zip(*zipped)
            else:
                zipped = zip(self.xValues, self.yValues)
                zipped.sort()
                self.xValues, self.yValues = zip(*zipped)

        if len(self.xTickLabels) > 0:
            if len(self.xTickLabelPoints) == 0:
                axis.set_xticks(self.xValues[0:len(self.xTickLabels)])
            else:
                axis.set_xticks(self.xTickLabelPoints)

            axis.set_xticklabels(self.xTickLabels, **self.xTickLabelProperties)

        if len(self.yTickLabels) > 0:
            if len(self.yTickLabelPoints) == 0:
                axis.set_yticks(self.yValues[0:len(self.yTickLabels)])
            else:
                axis.set_yticks(self.yTickLabelPoints)

            axis.set_yticklabels(self.yTickLabels, **self.yTickLabelProperties)

        self.drawErrorBars(axis)

    def drawErrorBars(self, axis, transform=None):
        """
        Draw error bars for the plot element using the plot element's
        `yMins`/`yMaxes` or `yErrors`. If `yMins` or `yMaxes` have non-zero
        length, they are used to draw error bars that can be
        asymmetric. Otherwise, `yErrors` are used to draw symmetric error bars.

        If some transform should be applied to the error bars, it is provided
        by the caller with the `transform` parameter.
        """

        errorBarKeywords = {}
        if  self.errorBarColor is not None:
            errorBarKeywords["ecolor"] = self.errorBarColor
        elif hasattr(self, "color"):
            errorBarKeywords["ecolor"] = self.color

        if self.errorBarCapsize is not None:
            errorBarKeywords["capsize"] = self.errorBarCapsize

        if self.errorBarLineWidth is not None:
            errorBarKeywords["linewidth"] = self.errorBarLineWidth

        if hasattr(self, "lineStyle"):
            errorBarKeywords["linestyle"] = self.lineStyle

        if transform:
            errorBarKeywords['transform'] = transform + axis.transData

        errorBarKeywords["fmt"] = None

        if len(self.yMins) > 0 and len(self.yMaxes) > 0:
            numYVals = len(self.yValues)
            yMin = [self.yValues[i] - self.yMins[i] for i in xrange(numYVals)]
            yMax = [self.yMaxes[i] - self.yValues[i] for i in xrange(numYVals)]

            errorBarKeywords["yerr"] = [yMin, yMax]

            axis.errorbar(self.xValues, self.yValues, **errorBarKeywords)
        elif len(self.yErrors) > 0:
            errorBarKeywords["yerr"] = self.yErrors
            axis.errorbar(self.xValues, self.yValues, **errorBarKeywords)

<|endoftext|>