<commit_before># -*- coding: utf-8 -*-

"""
JSON 2 HTML Converter
=====================

(c) Varun Malhotra 2013
Source Code: https://github.com/softvar/json2html

Contributors:
-------------
1. Michel Müller (@muellermichel), https://github.com/softvar/json2html/pull/2
2. Daniel Lekic (@lekic), https://github.com/softvar/json2html/pull/17

LICENSE: MIT
--------
"""

import sys
from collections import OrderedDict

if sys.version_info[:2] < (2, 7):
    import simplejson as json
else:
    import json

if sys.version_info[:2] < (3, 0):
    text = unicode
    text_types = [unicode, str]
else:
    text = str
    text_types = [str]

class Json2Html:

    def convert(self, **kwargs):
        """
            Convert JSON to HTML Table format
        """

        # table attributes such as class, id, data-attr-*, etc.
        # eg: table_attributes = 'class = "table table-bordered sortable"'
        global table_attributes
        table_attributes = ''

        if 'table_attributes' in kwargs:
            table_attributes = kwargs['table_attributes']
        else:
            # by default HTML table border
            table_attributes = 'border="1"'

        inputted_json = None
        if 'json' in kwargs and kwargs['json']:
            self.json_input = kwargs['json']
            if type(self.json_input) in text_types:
                inputted_json = json.loads(self.json_input, object_pairs_hook=OrderedDict)
            else:
                inputted_json = self.json_input
        else:
            raise ValueError("Please use json2html's convert function with a keyword argument 'json' - e.g. `json2html.convert(json={\"hello\":\"world!\"})`")

        if not isinstance(inputted_json, dict):
            raise ValueError(
                "Sorry, but json2html requires a dict as the top level object of your JSON. Your toplevel object is a %s" %(
                    str(type(inputted_json))
                )
            )

        return self.iter_json(inputted_json)

    def column_headers_from_list_of_dicts(self, inputted_json):
        """
            If suppose some key has array of objects and all the keys are same,
            instead of creating a new row for each such entry,
            club such values, thus it makes more sense and more readable table.

            @example:
                jsonObject = {
                    "sampleData": [
                        {"a":1, "b":2, "c":3},
                        {"a":5, "b":6, "c":7}
                    ]
                }
                OUTPUT:
                _____________________________
                |               |   |   |   |
                |               | a | c | b |
                |   sampleData  |---|---|---|
                |               | 1 | 3 | 2 |
                |               | 5 | 7 | 6 |
                -----------------------------

            @contributed by: @muellermichel
        """

        if not inputted_json or not isinstance(inputted_json[0], dict):
            return None

        column_headers = inputted_json[0].keys()

        for entry in inputted_json:
            if not isinstance(entry, dict) or (len(entry.keys()) != len(column_headers)):
                return None
            for header in column_headers:
                if header not in entry:
                    return None
        return column_headers

    def iter_json(self, inputted_json):
        """
            Iterate over the JSON and process it
            to generate the super awesome HTML Table format
        """

        def markup(entry):
            """
                Check for each value corresponding to its key
                and return accordingly
            """
            if type(entry) in text_types:
                return text(entry)
            if isinstance(entry, int) or isinstance(entry, float):
                return str(entry)
            if isinstance(entry, list) and len(entry) == 0:
                return ''
            if isinstance(entry, list):
                list_markup = '

'
                list_markup += '

'.join([markup(child) for child in entry])
                list_markup += '

'
                return list_markup
            if isinstance(entry, dict):
                return self.iter_json(entry)

            # safety: don't do recursion over anything that we don't know about
            # - iteritems() will most probably fail
            return ''

        converted_output = ''

        global table_attributes
        table_init_markup = "
" % table_attributes
        converted_output += table_init_markup

        for k, v in inputted_json.items():
            converted_output += '
' + markup(k) + '	'

            if v is None:
                v = text("")
            if isinstance(v, list):
                column_headers = self.column_headers_from_list_of_dicts(v)
                if column_headers is not None:
                    converted_output += ''
                    converted_output += table_init_markup
                    converted_output += '
' + '	'.join(column_headers) + '
'
                    for list_entry in v:
                        converted_output += '
'
                        converted_output += '	'.join([markup(list_entry[column_header]) for column_header in
                                                             column_headers])
                        converted_output += '
'

                    converted_output += '

'
                    continue
            converted_output += '' + markup(v) + '
'
        converted_output += '
'

        return converted_output

json2html = Json2Html()
<commit_msg>fixed imports for python 2.6<commit_after># -*- coding: utf-8 -*-

"""
JSON 2 HTML Converter
=====================

(c) Varun Malhotra 2013
Source Code: https://github.com/softvar/json2html

Contributors:
-------------
1. Michel Müller (@muellermichel), https://github.com/softvar/json2html/pull/2
2. Daniel Lekic (@lekic), https://github.com/softvar/json2html/pull/17

LICENSE: MIT
--------
"""

import sys

if sys.version_info[:2] < (2, 7):
    from ordereddict import OrderedDict
    import simplejson as json
else:
    from collections import OrderedDict
    import json

if sys.version_info[:2] < (3, 0):
    text = unicode
    text_types = [unicode, str]
else:
    text = str
    text_types = [str]

class Json2Html:

    def convert(self, **kwargs):
        """
            Convert JSON to HTML Table format
        """

        # table attributes such as class, id, data-attr-*, etc.
        # eg: table_attributes = 'class = "table table-bordered sortable"'
        global table_attributes
        table_attributes = ''

        if 'table_attributes' in kwargs:
            table_attributes = kwargs['table_attributes']
        else:
            # by default HTML table border
            table_attributes = 'border="1"'

        inputted_json = None
        if 'json' in kwargs and kwargs['json']:
            self.json_input = kwargs['json']
            if type(self.json_input) in text_types:
                inputted_json = json.loads(self.json_input, object_pairs_hook=OrderedDict)
            else:
                inputted_json = self.json_input
        else:
            raise ValueError("Please use json2html's convert function with a keyword argument 'json' - e.g. `json2html.convert(json={\"hello\":\"world!\"})`")

        if not isinstance(inputted_json, dict):
            raise ValueError(
                "Sorry, but json2html requires a dict as the top level object of your JSON. Your toplevel object is a %s" %(
                    str(type(inputted_json))
                )
            )

        return self.iter_json(inputted_json)

    def column_headers_from_list_of_dicts(self, inputted_json):
        """
            If suppose some key has array of objects and all the keys are same,
            instead of creating a new row for each such entry,
            club such values, thus it makes more sense and more readable table.

            @example:
                jsonObject = {
                    "sampleData": [
                        {"a":1, "b":2, "c":3},
                        {"a":5, "b":6, "c":7}
                    ]
                }
                OUTPUT:
                _____________________________
                |               |   |   |   |
                |               | a | c | b |
                |   sampleData  |---|---|---|
                |               | 1 | 3 | 2 |
                |               | 5 | 7 | 6 |
                -----------------------------

            @contributed by: @muellermichel
        """

        if not inputted_json or not isinstance(inputted_json[0], dict):
            return None

        column_headers = inputted_json[0].keys()

        for entry in inputted_json:
            if not isinstance(entry, dict) or (len(entry.keys()) != len(column_headers)):
                return None
            for header in column_headers:
                if header not in entry:
                    return None
        return column_headers

    def iter_json(self, inputted_json):
        """
            Iterate over the JSON and process it
            to generate the super awesome HTML Table format
        """

        def markup(entry):
            """
                Check for each value corresponding to its key
                and return accordingly
            """
            if type(entry) in text_types:
                return text(entry)
            if isinstance(entry, int) or isinstance(entry, float):
                return str(entry)
            if isinstance(entry, list) and len(entry) == 0:
                return ''
            if isinstance(entry, list):
                list_markup = '

'
                list_markup += '

'.join([markup(child) for child in entry])
                list_markup += '

'
                return list_markup
            if isinstance(entry, dict):
                return self.iter_json(entry)

            # safety: don't do recursion over anything that we don't know about
            # - iteritems() will most probably fail
            return ''

        converted_output = ''

        global table_attributes
        table_init_markup = "
" % table_attributes
        converted_output += table_init_markup

        for k, v in inputted_json.items():
            converted_output += '
' + markup(k) + '	'

            if v is None:
                v = text("")
            if isinstance(v, list):
                column_headers = self.column_headers_from_list_of_dicts(v)
                if column_headers is not None:
                    converted_output += ''
                    converted_output += table_init_markup
                    converted_output += '
' + '	'.join(column_headers) + '
'
                    for list_entry in v:
                        converted_output += '
'
                        converted_output += '	'.join([markup(list_entry[column_header]) for column_header in
                                                             column_headers])
                        converted_output += '
'

                    converted_output += '

'
                    continue
            converted_output += '' + markup(v) + '
'
        converted_output += '
'

        return converted_output

json2html = Json2Html()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
#   Copyright [2012] [Patrick Ancillotti]
#   Copyright [2011] [Jason Kölker]
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#

import logging
import traceback
import itertools

import anyjson as json

class JSONFormatter(logging.Formatter):

    def __init__(self, fmt=None, datefmt=None):
        self.datefmt = datefmt

    def formatException(self, ei, strip_newlines=True):
        lines = traceback.format_exception(*ei)
        if strip_newlines:
            lines = [itertools.ifilter(lambda x: x,
                                      line.rstrip().splitlines())
                    for line in lines]
            lines = list(itertools.chain(*lines))
        return lines

    def format(self, record):
        message = {'message': record.getMessage(),
                   'asctime': self.formatTime(record, self.datefmt),
                   'name': record.name,
                   'msg': record.msg,
                   'args': record.args,
                   'levelname': record.levelname,
                   'levelno': record.levelno,
                   'pathname': record.pathname,
                   'filename': record.filename,
                   'module': record.module,
                   'lineno': record.lineno,
                   'funcname': record.funcName,
                   'created': record.created,
                   'msecs': record.msecs,
                   'relative_created': record.relativeCreated,
                   'thread': record.thread,
                   'thread_name': record.threadName,
                   'process_name': record.processName,
                   'process': record.process,
                   'traceback': None}

        if hasattr(record, 'extra'):
            message['extra'] = record.extra

        if record.exc_info:
            message['traceback'] = self.formatException(record.exc_info)

        return json.dumps(message)

class JSONFormatterSimple(JSONFormatter):

    def format(self, record):
        message = {'message': record.getMessage(),
                   'asctime': self.formatTime(record, self.datefmt),
                   'levelname': record.levelname}

        return json.dumps(message)
<commit_msg>Added extras back into JSONFormatterSimple<commit_after># -*- coding: utf-8 -*-
#
#   Copyright [2012] [Patrick Ancillotti]
#   Copyright [2011] [Jason Kölker]
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#

import logging
import traceback
import itertools

import anyjson as json

class JSONFormatter(logging.Formatter):

    def __init__(self, fmt=None, datefmt=None):
        self.datefmt = datefmt

    def formatException(self, ei, strip_newlines=True):
        lines = traceback.format_exception(*ei)
        if strip_newlines:
            lines = [itertools.ifilter(lambda x: x,
                                      line.rstrip().splitlines())
                    for line in lines]
            lines = list(itertools.chain(*lines))
        return lines

    def format(self, record):
        message = {'message': record.getMessage(),
                   'asctime': self.formatTime(record, self.datefmt),
                   'name': record.name,
                   'msg': record.msg,
                   'args': record.args,
                   'levelname': record.levelname,
                   'levelno': record.levelno,
                   'pathname': record.pathname,
                   'filename': record.filename,
                   'module': record.module,
                   'lineno': record.lineno,
                   'funcname': record.funcName,
                   'created': record.created,
                   'msecs': record.msecs,
                   'relative_created': record.relativeCreated,
                   'thread': record.thread,
                   'thread_name': record.threadName,
                   'process_name': record.processName,
                   'process': record.process,
                   'traceback': None}

        if hasattr(record, 'extra'):
            message['extra'] = record.extra

        if record.exc_info:
            message['traceback'] = self.formatException(record.exc_info)

        return json.dumps(message)

class JSONFormatterSimple(JSONFormatter):

    def format(self, record):
        message = {'message': record.getMessage(),
                   'asctime': self.formatTime(record, self.datefmt),
                   'levelname': record.levelname}

        if hasattr(record, 'extra'):
            message['extra'] = record.extra

        return json.dumps(message)
<|endoftext|>


<commit_before>"""
Grabs a list of modules from drupal's online list of modules.
"""

import sys
sys.path.append("../../")

from bs4 import BeautifulSoup
import re
import requests

try:
    what = sys.argv[1]
except IndexError:
    print("Generates ONLY the module names to import from composer. See notes \
        for getting modules")
    print("Usage: %s (all|theme)" % sys.argv[0])
    sys.exit()

what = "" if what == "all" else what

# SS displays 16 modules per page :P
per_page = 16
base_url = 'http://addons.silverstripe.org/add-ons?search=&type=%s&sort=downloads&start=%s'
module_names = []
for page in xrange(0, 1000):
    start = page * per_page
    response = requests.get(base_url % (what, start))
    soup = BeautifulSoup(response.text)

    links = soup.select('.table-addons td:first-child a')

    if len(links) < per_page:
        if len(links) == 0:
            break
        else:
            pass

    for link in links:
        print link.contents[0]
<commit_msg>Update generate-composer.py<commit_after>"""
Grabs a list of modules from SilverStripe's online list of modules.
"""

import sys
sys.path.append("../../")

from bs4 import BeautifulSoup
import re
import requests

try:
    what = sys.argv[1]
except IndexError:
    print("Generates ONLY the module names to import from composer. See notes \
        for getting modules")
    print("Usage: %s (all|theme)" % sys.argv[0])
    sys.exit()

what = "" if what == "all" else what

# SS displays 16 modules per page :P
per_page = 16
base_url = 'http://addons.silverstripe.org/add-ons?search=&type=%s&sort=downloads&start=%s'
module_names = []
for page in xrange(0, 1000):
    start = page * per_page
    response = requests.get(base_url % (what, start))
    soup = BeautifulSoup(response.text)

    links = soup.select('.table-addons td:first-child a')

    if len(links) < per_page:
        if len(links) == 0:
            break
        else:
            pass

    for link in links:
        print link.contents[0]
<|endoftext|>


<commit_before>import logging

from django.conf import settings
from django import template
from django.db import models
from django.utils.encoding import smart_str
from django.utils.safestring import mark_safe
from django.template.defaultfilters import stringfilter

from ella.core.models import Listing, Category
from ella.core.cache.utils import get_cached_object
from ella.core.box import Box

log = logging.getLogger('ella.core.templatetags')
register = template.Library()

class ListingNode(template.Node):
    def __init__(self, var_name, parameters, parameters_to_resolve):
        self.var_name = var_name
        self.parameters = parameters
        self.parameters_to_resolve = parameters_to_resolve
        self.resolved_parameters = self.parameters.copy()

    def resolve_parameter(self, key, context):
        value = self.parameters[key]
        if key == 'count':
            if type(value) in (str, unicode) and value.isdigit():
                return int(value)
        return template.Variable(value).resolve(context)

    def render(self, context):
        for key in self.parameters_to_resolve:
            #self.parameters[key] = template.Variable(self.parameters[key]).resolve(context)
            self.resolved_parameters[key] = self.resolve_parameter(key, context)
        if self.resolved_parameters.has_key('category') and \
            isinstance(self.resolved_parameters['category'], basestring):
            self.resolved_parameters['category'] = get_cached_object(Category, tree_path=self.resolved_parameters['category'], site__id=settings.SITE_ID)
        out = Listing.objects.get_listing(**self.resolved_parameters)

        context[self.var_name] = out
        return ''

@register.tag
def listing(parser, token):
    """
    Tag that will obtain listing of top objects for a given category and store them in context under given name.

    Usage::

        {% listing <limit>[ from <offset>][of <app.model>[, <app.model>[, ...]]][ for <category> ] [with children|descendents] as <result> %}

    Parameters:
        ==================================  ================================================
        Option                              Description
        ==================================  ================================================
        ``limit``                           Number of objects to retrieve.
        ``offset``                          Starting with number (1-based), starts from first
                                            if no offset specified.
        ``app.model``, ...                  List of allowed models, all if omitted.
        ``category``                        Category of the listing, all categories if not
                                            specified. Can be either string (tree path),
                                            or variable containing a Category object.
        ``children``                        Include items from direct subcategories.
        ``descendents``                     Include items from all descend subcategories.
        ``result``                          Store the resulting list in context under given
                                            name.
        ==================================  ================================================

    Examples::

        {% listing 10 of articles.article for "home_page" as obj_list %}
        {% listing 10 of articles.article for category as obj_list %}
        {% listing 10 of articles.article for category with children as obj_list %}
        {% listing 10 of articles.article for category with descendents as obj_list %}
        {% listing 10 from 10 of articles.article as obj_list %}
        {% listing 10 of articles.article, photos.photo as obj_list %}

    """
    var_name, parameters, parameters_to_resolve = listing_parse(token.split_contents())
    return ListingNode(var_name, parameters, parameters_to_resolve)

def listing_parse(input):
    params = {}
    params_to_resolve = []
    if len(input) < 4:
        raise template.TemplateSyntaxError, "%r tag argument should have at least 4 arguments" % input[0]
    o = 1
    # limit
    params['count'] = input[o]
    params_to_resolve.append('count')
    o = 2
    # offset
    if input[o] == 'from':
        params['offset'] = input[o + 1]
        params_to_resolve.append('offset')
        o = o + 2
    # from - models definition
    if input[o] == 'of':
        o = o + 1
        if 'for' in input:
            mc = input.index('for')
        elif 'as' in input:
            mc = input.index('as')
        if mc > 0:
            l = []
            for mod in ''.join(input[o:mc]).split(','):
                m = models.get_model(*mod.split('.'))
                if m is None:
                    raise template.TemplateSyntaxError, "%r tag cannot list objects of unknown model %r" % (input[0], mod)
                l.append(m)
            params['mods'] = l
        o = mc
    # for - category definition
    if input[o] == 'for':
        params['category'] = input[o + 1]
        params_to_resolve.append('category')
        o = o + 2
    # with
    if input[o] == 'with':
        o = o + 1
        if input[o] == 'children':
            params['children'] = Listing.objects.IMMEDIATE
        elif input[o] == 'descendents':
            params['children'] = Listing.objects.ALL
        else:
            raise template.TemplateSyntaxError, "%r tag's argument 'with' required specification (with children|with descendents)" % input[0]
        o = o + 1

    # as
    if input[o] == 'as':
        var_name = input[o + 1]
    else:
        raise template.TemplateSyntaxError, "%r tag requires 'as' argument" % input[0]

    return var_name, params, params_to_resolve

class EmptyNode(template.Node):
    def render(self, context):
        return u''

class ObjectNotFoundOrInvalid(Exception): pass

class BoxNode(template.Node):

    def __init__(self, box_type, nodelist, model=None, lookup=None, var_name=None):
        self.box_type, self.nodelist, self.var_name, self.lookup, self.model = box_type, nodelist, var_name, lookup, model

    def get_obj(self, context=None):
        if self.model and self.lookup:
            if context:
                try:
                    lookup_val = template.Variable(self.lookup[1]).resolve(context)
                except template.VariableDoesNotExist:
                    lookup_val = self.lookup[1]
            else:
                lookup_val = self.lookup[1]

            try:
                obj = get_cached_object(self.model, **{self.lookup[0] : lookup_val})
            except models.ObjectDoesNotExist, e:
                log.error('BoxNode: %s (%s : %s)' % (str(e), self.lookup[0], lookup_val))
                raise ObjectNotFoundOrInvalid()
            except AssertionError, e:
                log.error('BoxNode: %s (%s : %s)' % (str(e), self.lookup[0], lookup_val))
                raise ObjectNotFoundOrInvalid()
        else:
            if not context:
                raise ObjectNotFoundOrInvalid()
            try:
                obj = template.Variable(self.var_name).resolve(context)
            except template.VariableDoesNotExist, e:
                log.error('BoxNode: Template variable does not exist. var_name=%s' % self.var_name)
                raise ObjectNotFoundOrInvalid()
            if not obj:
                raise ObjectNotFoundOrInvalid()
        return obj

    def render(self, context):

        try:
            obj = self.get_obj(context)
        except ObjectNotFoundOrInvalid, e:
            return ''

        box = getattr(obj, 'box_class', Box)(obj, self.box_type, self.nodelist)

        if not box or not box.obj:
            log.warning('BoxNode: Box does not exists.')
            return ''

        # render the box itself
        box.prepare(context)

        # render the box
        result = box.render()

        return result

@register.tag('box')
def do_box(parser, token):
    """
    Tag Node representing our idea of a reusable box. It can handle multiple
    parameters in its body which will then be accessible via ``{{ box.params
    }}`` in the template being rendered.

    .. note::
        The inside of the box will be rendered only when redering the box in
        current context and the ``object`` template variable will be present
        and set to the target of the box.

    Author of any ``Model`` can specify it's own ``box_class`` which enables
    custom handling of some content types (boxes for polls for example need
    some extra information to render properly).

    Boxes, same as :ref:`core-views`, look for most specific template for a given
    object an only fall back to more generic template if the more specific one
    doesn't exist. The list of templates it looks for:

    * ``box/category/<tree_path>/content_type/<app>.<model>/<slug>/<box_name>.html``
    * ``box/category/<tree_path>/content_type/<app>.<model>/<box_name>.html``
    * ``box/category/<tree_path>/content_type/<app>.<model>/box.html``
    * ``box/content_type/<app>.<model>/<slug>/<box_name>.html``
    * ``box/content_type/<app>.<model>/<box_name>.html``
    * ``box/content_type/<app>.<model>/box.html``
    * ``box/<box_name>.html``
    * ``box/box.html``

    .. note::
        Since boxes work for all models (and not just ``Publishable`` subclasses),
        some template names don't exist for some model classes, for example
        ``Photo`` model doesn't have a link to ``Category`` so that cannot be used.

    Boxes are always rendered in current context with added variables:

    * ``object`` - object being represented
    * ``box`` - instance of ``ella.core.box.Box``

    Usage::

        {% box <boxtype> for <app.model> with <field> <value> %}
            param_name: value
            param_name_2: {{ some_var }}
        {% endbox %}

        {% box <boxtype> for <var_name> %}
            ...
        {% endbox %}

    Parameters:

        ==================================  ================================================
        Option                              Description
        ==================================  ================================================
        ``boxtype``                         Name of the box to use
        ``app.model``                       Model class to use
        ``field``                           Field on which to do DB lookup
        ``value``                           Value for DB lookup
        ``var_name``                        Template variable to get the instance from
        ==================================  ================================================

    Examples::

        {% box home_listing for articles.article with slug "some-slug" %}{% endbox %}

        {% box home_listing for articles.article with pk object_id %}
            template_name : {{object.get_box_template}}
        {% endbox %}

        {% box home_listing for article %}{% endbox %}
    """
    bits = token.split_contents()

    nodelist = parser.parse(('end' + bits[0],))
    parser.delete_first_token()
    return _parse_box(nodelist, bits)

def _parse_box(nodelist, bits):
    # {% box BOXTYPE for var_name %}                {% box BOXTYPE for content.type with PK_FIELD PK_VALUE %}
    if (len(bits) != 4 or bits[2] != 'for') and (len(bits) != 7 or bits[2] != 'for' or bits[4] != 'with'):
        raise template.TemplateSyntaxError, "{% box BOXTYPE for content.type with FIELD VALUE %} or {% box BOXTYPE for var_name %}"

    if len(bits) == 4:
        # var_name
        return BoxNode(bits[1], nodelist, var_name=bits[3])
    else:
        model = models.get_model(*bits[3].split('.'))
        if model is None:
            return EmptyNode()

        return BoxNode(bits[1], nodelist, model=model, lookup=(smart_str(bits[5]), bits[6]))

class RenderNode(template.Node):
    def __init__(self, var):
        self.var = template.Variable(var)

    def render(self, context):
        try:
            text = self.var.resolve(context)
        except template.VariableDoesNotExist:
            return ''

        template_name = 'render-%s' % self.var
        return template.Template(text, name=template_name).render(context)

@register.tag('render')
def do_render(parser, token):
    """
    {% render some_var %}
    """
    bits = token.split_contents()

    if len(bits) != 2:
        raise template.TemplateSyntaxError()

    return RenderNode(bits[1])

@register.filter
@stringfilter
def ipblur(text): # brutalizer ;-)
    """ blurs IP address  """
    import re
    m = re.match(r'^(\d{1,3}\.\d{1,3}\.\d{1,3}\.)\d{1,3}.*', text)
    if not m:
        return text
    return '%sxxx' % m.group(1)

@register.filter
@stringfilter
def emailblur(email):
    "Obfuscates e-mail addresses - only @ and dot"
    return mark_safe(email.replace('@', '@').replace('.', '.'))

<commit_msg>simpler code is better code<commit_after>import logging

from django.conf import settings
from django import template
from django.db import models
from django.utils.encoding import smart_str
from django.utils.safestring import mark_safe
from django.template.defaultfilters import stringfilter

from ella.core.models import Listing, Category
from ella.core.cache.utils import get_cached_object
from ella.core.box import Box

log = logging.getLogger('ella.core.templatetags')
register = template.Library()

class ListingNode(template.Node):
    def __init__(self, var_name, parameters, parameters_to_resolve):
        self.var_name = var_name
        self.parameters = parameters
        self.parameters_to_resolve = parameters_to_resolve
        self.resolved_parameters = self.parameters.copy()

    def resolve_parameter(self, key, context):
        value = self.parameters[key]
        if key == 'count':
            if type(value) in (str, unicode) and value.isdigit():
                return int(value)
        return template.Variable(value).resolve(context)

    def render(self, context):
        for key in self.parameters_to_resolve:
            #self.parameters[key] = template.Variable(self.parameters[key]).resolve(context)
            self.resolved_parameters[key] = self.resolve_parameter(key, context)
        if self.resolved_parameters.has_key('category') and \
            isinstance(self.resolved_parameters['category'], basestring):
            self.resolved_parameters['category'] = get_cached_object(Category, tree_path=self.resolved_parameters['category'], site__id=settings.SITE_ID)
        out = Listing.objects.get_listing(**self.resolved_parameters)

        context[self.var_name] = out
        return ''

@register.tag
def listing(parser, token):
    """
    Tag that will obtain listing of top objects for a given category and store them in context under given name.

    Usage::

        {% listing <limit>[ from <offset>][of <app.model>[, <app.model>[, ...]]][ for <category> ] [with children|descendents] as <result> %}

    Parameters:
        ==================================  ================================================
        Option                              Description
        ==================================  ================================================
        ``limit``                           Number of objects to retrieve.
        ``offset``                          Starting with number (1-based), starts from first
                                            if no offset specified.
        ``app.model``, ...                  List of allowed models, all if omitted.
        ``category``                        Category of the listing, all categories if not
                                            specified. Can be either string (tree path),
                                            or variable containing a Category object.
        ``children``                        Include items from direct subcategories.
        ``descendents``                     Include items from all descend subcategories.
        ``result``                          Store the resulting list in context under given
                                            name.
        ==================================  ================================================

    Examples::

        {% listing 10 of articles.article for "home_page" as obj_list %}
        {% listing 10 of articles.article for category as obj_list %}
        {% listing 10 of articles.article for category with children as obj_list %}
        {% listing 10 of articles.article for category with descendents as obj_list %}
        {% listing 10 from 10 of articles.article as obj_list %}
        {% listing 10 of articles.article, photos.photo as obj_list %}

    """
    var_name, parameters, parameters_to_resolve = listing_parse(token.split_contents())
    return ListingNode(var_name, parameters, parameters_to_resolve)

def listing_parse(input):
    params = {}
    params_to_resolve = []
    if len(input) < 4:
        raise template.TemplateSyntaxError, "%r tag argument should have at least 4 arguments" % input[0]
    o = 1
    # limit
    params['count'] = input[o]
    params_to_resolve.append('count')
    o = 2
    # offset
    if input[o] == 'from':
        params['offset'] = input[o + 1]
        params_to_resolve.append('offset')
        o = o + 2
    # from - models definition
    if input[o] == 'of':
        o = o + 1
        if 'for' in input:
            mc = input.index('for')
        elif 'as' in input:
            mc = input.index('as')
        if mc > 0:
            l = []
            for mod in ''.join(input[o:mc]).split(','):
                m = models.get_model(*mod.split('.'))
                if m is None:
                    raise template.TemplateSyntaxError, "%r tag cannot list objects of unknown model %r" % (input[0], mod)
                l.append(m)
            params['mods'] = l
        o = mc
    # for - category definition
    if input[o] == 'for':
        params['category'] = input[o + 1]
        params_to_resolve.append('category')
        o = o + 2
    # with
    if input[o] == 'with':
        o = o + 1
        if input[o] == 'children':
            params['children'] = Listing.objects.IMMEDIATE
        elif input[o] == 'descendents':
            params['children'] = Listing.objects.ALL
        else:
            raise template.TemplateSyntaxError, "%r tag's argument 'with' required specification (with children|with descendents)" % input[0]
        o = o + 1

    # as
    if input[o] == 'as':
        var_name = input[o + 1]
    else:
        raise template.TemplateSyntaxError, "%r tag requires 'as' argument" % input[0]

    return var_name, params, params_to_resolve

class EmptyNode(template.Node):
    def render(self, context):
        return u''

class ObjectNotFoundOrInvalid(Exception): pass

class BoxNode(template.Node):

    def __init__(self, box_type, nodelist, model=None, lookup=None, var_name=None):
        self.box_type, self.nodelist, self.var_name, self.lookup, self.model = box_type, nodelist, var_name, lookup, model

    def get_obj(self, context):
        if self.model and self.lookup:
            try:
                lookup_val = template.Variable(self.lookup[1]).resolve(context)
            except template.VariableDoesNotExist:
                lookup_val = self.lookup[1]

            try:
                obj = get_cached_object(self.model, **{self.lookup[0] : lookup_val})
            except models.ObjectDoesNotExist, e:
                log.error('BoxNode: %s (%s : %s)' % (str(e), self.lookup[0], lookup_val))
                raise ObjectNotFoundOrInvalid()
            except AssertionError, e:
                log.error('BoxNode: %s (%s : %s)' % (str(e), self.lookup[0], lookup_val))
                raise ObjectNotFoundOrInvalid()
        else:
            try:
                obj = template.Variable(self.var_name).resolve(context)
            except template.VariableDoesNotExist, e:
                log.error('BoxNode: Template variable does not exist. var_name=%s' % self.var_name)
                raise ObjectNotFoundOrInvalid()
            if not obj:
                raise ObjectNotFoundOrInvalid()
        return obj

    def render(self, context):

        try:
            obj = self.get_obj(context)
        except ObjectNotFoundOrInvalid, e:
            return ''

        box = getattr(obj, 'box_class', Box)(obj, self.box_type, self.nodelist)

        if not box or not box.obj:
            log.warning('BoxNode: Box does not exists.')
            return ''

        # render the box itself
        box.prepare(context)

        # render the box
        result = box.render()

        return result

@register.tag('box')
def do_box(parser, token):
    """
    Tag Node representing our idea of a reusable box. It can handle multiple
    parameters in its body which will then be accessible via ``{{ box.params
    }}`` in the template being rendered.

    .. note::
        The inside of the box will be rendered only when redering the box in
        current context and the ``object`` template variable will be present
        and set to the target of the box.

    Author of any ``Model`` can specify it's own ``box_class`` which enables
    custom handling of some content types (boxes for polls for example need
    some extra information to render properly).

    Boxes, same as :ref:`core-views`, look for most specific template for a given
    object an only fall back to more generic template if the more specific one
    doesn't exist. The list of templates it looks for:

    * ``box/category/<tree_path>/content_type/<app>.<model>/<slug>/<box_name>.html``
    * ``box/category/<tree_path>/content_type/<app>.<model>/<box_name>.html``
    * ``box/category/<tree_path>/content_type/<app>.<model>/box.html``
    * ``box/content_type/<app>.<model>/<slug>/<box_name>.html``
    * ``box/content_type/<app>.<model>/<box_name>.html``
    * ``box/content_type/<app>.<model>/box.html``
    * ``box/<box_name>.html``
    * ``box/box.html``

    .. note::
        Since boxes work for all models (and not just ``Publishable`` subclasses),
        some template names don't exist for some model classes, for example
        ``Photo`` model doesn't have a link to ``Category`` so that cannot be used.

    Boxes are always rendered in current context with added variables:

    * ``object`` - object being represented
    * ``box`` - instance of ``ella.core.box.Box``

    Usage::

        {% box <boxtype> for <app.model> with <field> <value> %}
            param_name: value
            param_name_2: {{ some_var }}
        {% endbox %}

        {% box <boxtype> for <var_name> %}
            ...
        {% endbox %}

    Parameters:

        ==================================  ================================================
        Option                              Description
        ==================================  ================================================
        ``boxtype``                         Name of the box to use
        ``app.model``                       Model class to use
        ``field``                           Field on which to do DB lookup
        ``value``                           Value for DB lookup
        ``var_name``                        Template variable to get the instance from
        ==================================  ================================================

    Examples::

        {% box home_listing for articles.article with slug "some-slug" %}{% endbox %}

        {% box home_listing for articles.article with pk object_id %}
            template_name : {{object.get_box_template}}
        {% endbox %}

        {% box home_listing for article %}{% endbox %}
    """
    bits = token.split_contents()

    nodelist = parser.parse(('end' + bits[0],))
    parser.delete_first_token()
    return _parse_box(nodelist, bits)

def _parse_box(nodelist, bits):
    # {% box BOXTYPE for var_name %}                {% box BOXTYPE for content.type with PK_FIELD PK_VALUE %}
    if (len(bits) != 4 or bits[2] != 'for') and (len(bits) != 7 or bits[2] != 'for' or bits[4] != 'with'):
        raise template.TemplateSyntaxError, "{% box BOXTYPE for content.type with FIELD VALUE %} or {% box BOXTYPE for var_name %}"

    if len(bits) == 4:
        # var_name
        return BoxNode(bits[1], nodelist, var_name=bits[3])
    else:
        model = models.get_model(*bits[3].split('.'))
        if model is None:
            return EmptyNode()

        return BoxNode(bits[1], nodelist, model=model, lookup=(smart_str(bits[5]), bits[6]))

class RenderNode(template.Node):
    def __init__(self, var):
        self.var = template.Variable(var)

    def render(self, context):
        try:
            text = self.var.resolve(context)
        except template.VariableDoesNotExist:
            return ''

        template_name = 'render-%s' % self.var
        return template.Template(text, name=template_name).render(context)

@register.tag('render')
def do_render(parser, token):
    """
    {% render some_var %}
    """
    bits = token.split_contents()

    if len(bits) != 2:
        raise template.TemplateSyntaxError()

    return RenderNode(bits[1])

@register.filter
@stringfilter
def ipblur(text): # brutalizer ;-)
    """ blurs IP address  """
    import re
    m = re.match(r'^(\d{1,3}\.\d{1,3}\.\d{1,3}\.)\d{1,3}.*', text)
    if not m:
        return text
    return '%sxxx' % m.group(1)

@register.filter
@stringfilter
def emailblur(email):
    "Obfuscates e-mail addresses - only @ and dot"
    return mark_safe(email.replace('@', '@').replace('.', '.'))

<|endoftext|>


<commit_before>from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from future import standard_library
standard_library.install_aliases()
from builtins import range
from builtins import *
import logging
import multiprocessing as mp
import pandas as pd
from uuid import UUID

import emission.core.get_database as edb
import emission.net.usercache.abstract_usercache as enua
import emission.storage.timeseries.aggregate_timeseries as estag
import emission.storage.decorations.user_queries as esdu
import emission.pipeline.intake_stage as epi

def get_split_uuid_lists(n_splits):
    get_count = lambda u: enua.UserCache.getUserCache(u).getMessageCount()

    """
    This is the count of messages in the usercache. While we can use it for the
    scheduling, it is just a hint, because having a zero count here is no guarantee
    of zero count in the processing. And in particular, having a zero count here is
    no indication that there are no pending entries in the long-term cache. I think
    that's why we used to have the long-term cache and the user cache uuid list separately.

    That technique is no longer feasible because it requires coordination after the
    usercache is processed instead of parallelizing the entire pipeline.

    In general, this should be a pretty good hint, but I am not sure that it is perfect,
    and I am not ready to try this out weeks before the big deployment, with one day's
    testing. I also don't want to use UserCache.getMessage() since it returns the entire
    list instead of just the count. So we use the count as a hint and don't filter out
    users based on that.
    """

    sel_uuids = esdu.get_all_uuids()
    sel_jobs = [(u, get_count(u)) for u in sel_uuids]
    # non_zero_jobs = [j for j in sel_jobs if j[1] !=0 ]
    # Not filtering for now
    non_zero_jobs = sel_jobs
    logging.debug("sel_uuids = %s, sel_jobs = %s, non_zero_jobs = %s" %
                  (len(sel_uuids), len(sel_jobs), len(non_zero_jobs)))

    non_zero_jobs_df = pd.DataFrame(non_zero_jobs, columns=['user_id', 'count']).sort_values(by="count")
    ret_splits = []
    for i in range(0, n_splits):
        ret_splits.append([])

    col = 0
    for i, nzj in enumerate(non_zero_jobs_df.to_dict('records')):
        ret_splits[col].append(nzj['user_id'])
        col = col + 1
        if col == n_splits:
            logging.debug("reached n_splits, setting to zero")
            col = 0
    logging.debug("Split values are %s" % ret_splits)
    return ret_splits

def dispatch(split_lists):
    process_list = []
    for i, uuid_list in enumerate(split_lists):
        logging.debug("Dispatching list %s" % uuid_list)
        pid = i
        p = mp.Process(target=epi.run_intake_pipeline, args=(pid, uuid_list))
        logging.info("Created process %s to process %s list of size %s" %
                     (p, i, len(uuid_list)))
        p.start()
        process_list.append(p)

<commit_msg>Spawn processes instead of forking them<commit_after>from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from future import standard_library
standard_library.install_aliases()
from builtins import range
from builtins import *
import logging
import multiprocessing as mp
import pandas as pd
from uuid import UUID

import emission.core.get_database as edb
import emission.net.usercache.abstract_usercache as enua
import emission.storage.timeseries.aggregate_timeseries as estag
import emission.storage.decorations.user_queries as esdu
import emission.pipeline.intake_stage as epi

def get_split_uuid_lists(n_splits):
    get_count = lambda u: enua.UserCache.getUserCache(u).getMessageCount()

    """
    This is the count of messages in the usercache. While we can use it for the
    scheduling, it is just a hint, because having a zero count here is no guarantee
    of zero count in the processing. And in particular, having a zero count here is
    no indication that there are no pending entries in the long-term cache. I think
    that's why we used to have the long-term cache and the user cache uuid list separately.

    That technique is no longer feasible because it requires coordination after the
    usercache is processed instead of parallelizing the entire pipeline.

    In general, this should be a pretty good hint, but I am not sure that it is perfect,
    and I am not ready to try this out weeks before the big deployment, with one day's
    testing. I also don't want to use UserCache.getMessage() since it returns the entire
    list instead of just the count. So we use the count as a hint and don't filter out
    users based on that.
    """

    sel_uuids = esdu.get_all_uuids()
    sel_jobs = [(u, get_count(u)) for u in sel_uuids]
    # non_zero_jobs = [j for j in sel_jobs if j[1] !=0 ]
    # Not filtering for now
    non_zero_jobs = sel_jobs
    logging.debug("sel_uuids = %s, sel_jobs = %s, non_zero_jobs = %s" %
                  (len(sel_uuids), len(sel_jobs), len(non_zero_jobs)))

    non_zero_jobs_df = pd.DataFrame(non_zero_jobs, columns=['user_id', 'count']).sort_values(by="count")
    ret_splits = []
    for i in range(0, n_splits):
        ret_splits.append([])

    col = 0
    for i, nzj in enumerate(non_zero_jobs_df.to_dict('records')):
        ret_splits[col].append(nzj['user_id'])
        col = col + 1
        if col == n_splits:
            logging.debug("reached n_splits, setting to zero")
            col = 0
    logging.debug("Split values are %s" % ret_splits)
    return ret_splits

def dispatch(split_lists):
    ctx = mp.get_context('spawn')
    process_list = []
    for i, uuid_list in enumerate(split_lists):
        logging.debug("Dispatching list %s" % uuid_list)
        pid = i
        p = ctx.Process(target=epi.run_intake_pipeline, args=(pid, uuid_list))
        logging.info("Created process %s to process %s list of size %s" %
                     (p, i, len(uuid_list)))
        p.start()
        process_list.append(p)

<|endoftext|>


<commit_before>import luigi
from luigi import configuration
from luigi.contrib.pig import PigJobTask
from luigi.s3 import S3Target, S3PathTask

from amazon_web_service.luigi.emr_client import InitializeEmrCluster
from amazon_web_service.luigi.emr_client import TerminateEmrCluster

from amazon_web_service.luigi.emr_client import EmrPigTask

import logging
logger = logging.getLogger('luigi-interface')

import inspect
import os
import re

"""
   To run:
      Required Parameters:
         --project-root: The absolute path to your project. No trailing slash.

   Optional Parameters
      --output-root-path: The root location where output will be stored. No trailing slash.
      --input-root-path: The root loation for all input files. No trailing slash.

   Ex:
      python wordcount_pig.py --local-scheduler --project-root <Absolute path to your project> --input-root-path <Absolute path to input files> --output-root-path <Absolute path to output files>

"""

# Number of reduce task slots per m1.xlarge instance
NUM_REDUCE_SLOTS_PER_MACHINE = 3

def create_task_s3_target(root_path, task_str):
   """
   Helper method for easily creating S3Targets
   """
   return S3Target('%s/%s' % (root_path, task_str))

def create_task_local_target(root_path, task_str):
   """
   Helper method for easily creating local targets
   """
   return luigi.LocalTarget("%s/%s" % (root_path, task_str))

class LocalInputFile(luigi.Task):
   """
   A Task wrapping up a Target object
   """
   path = luigi.Parameter()

   def output(self):
      """
      Retun the Target path
      """
      return luigi.LocalTarget(self.path)

class PigscriptTask(PigJobTask):

   # The root path to where input data is located
   input_root_path = luigi.Parameter()

   # The root path to where output data will be written, a local or S3 path
   output_root_path = luigi.Parameter()

   # The absolute path to the root of the pigscript directory
   project_root = luigi.Parameter()

   # The cluster size to use for running job
   cluster_size = luigi.IntParameter(default=10)

   # A parameter to allow the job to run locally
   run_locally = luigi.BoolParameter(default=False)

   #
   root_path = None

   def pig_env_vars(self):
      return { "PIG_CLASSPATH":'%(root)s/.:%(root)s/lib-cluster/*:%(root)s/lib-pig/*:%(root)s/jython/jython.jar' % \
            {'root': self._get_root_path()}
   }

   def pig_properties(self):
      """
      Define pig specific properties that will be passed to all of the subclasses of
      PigTaskConfiguration
      """

      props = {
         'pig.logfile': self._logfile(),
         'pig.exec.reducers.bytes.per.reducer':268435456,
         'fs.default.name':'file:///',
         'pig.temp.dir':'/tmp/',
      }

      # Add s3 properties if running remotely
      if not self.run_locally:
         props.update({
            'fs.s3.impl':'org.apache.hadoop.fs.s3native.NativeS3FileSystem',
            'fs.s3n.awsAccessKeyId':self._get_aws_access_key(),
            'fs.s3n.awsSecretAccessKey':self._get_aws_secret_access_key(),
            'fs.s3.awsAccessKeyId':self._get_aws_access_key(),
            'fs.s3.awsSecretAccessKey':self._get_aws_secret_access_key(),
         })

      return props

   def pig_parameters(self):
      """
      Define pramaters that will be passed to all the subclasses of PigTaskConfiguration
      """

      # Parameters required to run the job locally
      params = {}

      # Add s3 properties if running remotely
      if not self.run_locally:
         params.update({
            'aws_access_key_id':self._get_aws_access_key(),
            'AWS_ACCESS_KEY_ID':self._get_aws_access_key(),
            'AWS_ACCESS_KEY':self._get_aws_access_key(),
            'aws_secret_acces_key':self._get_aws_secret_access_key(),
            'AWS_SECRET_KEY':self._get_aws_secret_access_key(),
            'AWS_SECRET_ACCESS_KEY':self._get_aws_secret_access_key(),
         })

      # Add any parameters defined in the specific task
      params.update(self.parameters())

      return params

   def parameters(self):
      """
      Dictionary of parameters that should be set for the Pig job.

      Ex::
         return { 'YOUR_PARAM_NAME':'Your param value' }
      """

      return {}

   def pig_options(self):
      """
      Define the options that will be passed all the subclasses of PigTaskConfiguration
      """

      # Set the path to the log4j conf file
      options = ['-log4jconf', self._log4j_conf()]

      # Run the job locall if the run-locally option is set
      if self.run_locally:
         options.extend(['-x', 'local'])

      return options

   def pig_script_path(self):
      """
      Return the absolute path to the pig script to run
      """

      return "%s/pigscripts/%s" % (self.project_root, self.script())

   def script(self):
      """
      Return the name of the Pig script to run
      """

      raise NotImplementedError("subclass should define script")

   def output(self):
      """
      Return the output path of the script
      """

      return self.script_output()

   def script_output(self):
      """

      List of output paths generated by the pigscript

      Ex::
         return ['output_path']
      """

      raise NotImplementedError("subclass should define the script output path")

   def books_file(self):
      """
      Return 'aws_access_key_id' from client.cfg
      """
      return "%s/%s" % (self.input_root_path, 'books/MobyDick.txt')

   def create_s3_target(self, task_str):
      return create_task_s3_target(self.output_root_path, task_str)

   def create_local_target(self, task_str):
      return create_task_local_target(self.output_root_path, task_str)

   def _get_aws_access_key(self):
      """
      Return 'aws_access_key_id' from client.cfg
      """

      return configuration.get_config().get('aws', 'aws_access_key_id')

   def _get_aws_secret_access_key(self):
      """
      Return 'aws_secret_access_key' from client.cfg
      """

      return configuration.get_config().get('aws', 'aws_secret_access_key')

   def _log4j_conf(self):
      """
      Return the logging properties for log4j
      """

      return '%s/conf/log4j-cli-local-dev.properties' % (self.project_root);

   def _logfile(self):
      """
      The the location to store the pig logs
      """

      return '%s/logs/local-pig.log' % (self.project_root)

   def _get_root_path(self):
      """
      Return the root path of the project
      """

      if not self.root_path:
         self.root_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))

      return self.root_path

   def _s3_safe(self, s):
      """
      Ensure the S3 path specified is syntatically correct
      """
      return re.sub("[^0-9a-zA-Z]", '-', s)

class WordCount(PigscriptTask):

   def requires(self):
      """
      The requires method is how you build your dependency graph in Luigi.  Luigi will not
      run this task until all tasks returning in this list are complete.

      S3PathTask is a simple Luigi task that ensures some path already exists. As
      GenerateUserSignals is the first task in this Luigi pipeline it only requires that its
      expected input data can be found.
      """
      if self.run_locally:
         return [LocalInputFile(self.books_file())]
      else:
         return [S3PathTask(self.books_file())]

   def script_output(self):
      """
      The script_output method defines where output from the task will be stored.

      Luigi will check this output location before starting any tasks that depend on this task.
      """

      if self.run_locally:
         return [self.create_local_target('wordcount')]
      else:
         return [self.create_s3_target('wordcount')]

   def parameters(self):
      """
      This method defines the parameters that will be passed to the pig script when starting
      this pigscript.
      """

      return {
         'OUTPUT_ROOT_PATH': self.output_root_path,
         'INPUT_ROOT_PATH' : self.input_root_path,
      }

   def script(self):
      """
      The name of the pigscript to run
      """
      return "wordcount.pig"

class CreateEmrCluster(InitializeEmrCluster):
    """
    This task creates an EMR cluster for runnig the rest of the tasks on
    """

    output_root_path = luigi.Parameter()

    def output_token(self):
        return create_task_s3_target(self.output_root_path,
                                     self.__class__.__name__)

class RemoteWordCount(EmrPigTask):

   # The root path to where input data is located
   input_root_path = luigi.Parameter()

   # The root path to where output data will be written, a local or S3 path
   output_root_path = luigi.Parameter()

   def pig_args(self):
      args = []
      args.extend(['-p','INPUT_ROOT_PATH=%s' % self.input_root_path])
      args.extend(['-p','OUTPUT_ROOT_PATH=%s' % self.output_root_path])
      return args

   def output_token(self):

      return create_task_s3_target(self.output_root_path,
                                   self.__class__.__name__)

class ShutdownEmrCluster(TerminateEmrCluster):
    """
    This task terminates an EMR cluster
    """

    output_root_path = luigi.Parameter()

    def output_token(self):
        return create_task_s3_target(self.output_root_path,
                                     self.__class__.__name__)

if __name__ == "__main__":
   """
   We tell Luigi to run the last task in the task dependency graph.  Luigi will then
   work backwards to find any tasks with its requirements met and start from there.

   The first time this pipeline is run the only tasks with the requirements met will be
   the GenerateUserSignals and GenerateItemSignals tasks which will find their
   required input files in S3.
   """
   # luigi.run(main_task_cls=WordCount)
   luigi.run(main_task_cls=CreateEmrCluster)
   # luigi.run(main_task_cls=RemoteWordCount)
   #luigi.run(main_task_cls=ExampleShutdownEmrCluster)
<commit_msg>ZER - Changed the name of the shutdown task<commit_after>import luigi
from luigi import configuration
from luigi.contrib.pig import PigJobTask
from luigi.s3 import S3Target, S3PathTask

from amazon_web_service.luigi.emr_client import InitializeEmrCluster
from amazon_web_service.luigi.emr_client import TerminateEmrCluster

from amazon_web_service.luigi.emr_client import EmrPigTask

import logging
logger = logging.getLogger('luigi-interface')

import inspect
import os
import re

"""
   To run:
      Required Parameters:
         --project-root: The absolute path to your project. No trailing slash.

   Optional Parameters
      --output-root-path: The root location where output will be stored. No trailing slash.
      --input-root-path: The root loation for all input files. No trailing slash.

   Ex:
      python wordcount_pig.py --local-scheduler --project-root <Absolute path to your project> --input-root-path <Absolute path to input files> --output-root-path <Absolute path to output files>

"""

# Number of reduce task slots per m1.xlarge instance
NUM_REDUCE_SLOTS_PER_MACHINE = 3

def create_task_s3_target(root_path, task_str):
   """
   Helper method for easily creating S3Targets
   """
   return S3Target('%s/%s' % (root_path, task_str))

def create_task_local_target(root_path, task_str):
   """
   Helper method for easily creating local targets
   """
   return luigi.LocalTarget("%s/%s" % (root_path, task_str))

class LocalInputFile(luigi.Task):
   """
   A Task wrapping up a Target object
   """
   path = luigi.Parameter()

   def output(self):
      """
      Retun the Target path
      """
      return luigi.LocalTarget(self.path)

class PigscriptTask(PigJobTask):

   # The root path to where input data is located
   input_root_path = luigi.Parameter()

   # The root path to where output data will be written, a local or S3 path
   output_root_path = luigi.Parameter()

   # The absolute path to the root of the pigscript directory
   project_root = luigi.Parameter()

   # The cluster size to use for running job
   cluster_size = luigi.IntParameter(default=10)

   # A parameter to allow the job to run locally
   run_locally = luigi.BoolParameter(default=False)

   #
   root_path = None

   def pig_env_vars(self):
      return { "PIG_CLASSPATH":'%(root)s/.:%(root)s/lib-cluster/*:%(root)s/lib-pig/*:%(root)s/jython/jython.jar' % \
            {'root': self._get_root_path()}
   }

   def pig_properties(self):
      """
      Define pig specific properties that will be passed to all of the subclasses of
      PigTaskConfiguration
      """

      props = {
         'pig.logfile': self._logfile(),
         'pig.exec.reducers.bytes.per.reducer':268435456,
         'fs.default.name':'file:///',
         'pig.temp.dir':'/tmp/',
      }

      # Add s3 properties if running remotely
      if not self.run_locally:
         props.update({
            'fs.s3.impl':'org.apache.hadoop.fs.s3native.NativeS3FileSystem',
            'fs.s3n.awsAccessKeyId':self._get_aws_access_key(),
            'fs.s3n.awsSecretAccessKey':self._get_aws_secret_access_key(),
            'fs.s3.awsAccessKeyId':self._get_aws_access_key(),
            'fs.s3.awsSecretAccessKey':self._get_aws_secret_access_key(),
         })

      return props

   def pig_parameters(self):
      """
      Define pramaters that will be passed to all the subclasses of PigTaskConfiguration
      """

      # Parameters required to run the job locally
      params = {}

      # Add s3 properties if running remotely
      if not self.run_locally:
         params.update({
            'aws_access_key_id':self._get_aws_access_key(),
            'AWS_ACCESS_KEY_ID':self._get_aws_access_key(),
            'AWS_ACCESS_KEY':self._get_aws_access_key(),
            'aws_secret_acces_key':self._get_aws_secret_access_key(),
            'AWS_SECRET_KEY':self._get_aws_secret_access_key(),
            'AWS_SECRET_ACCESS_KEY':self._get_aws_secret_access_key(),
         })

      # Add any parameters defined in the specific task
      params.update(self.parameters())

      return params

   def parameters(self):
      """
      Dictionary of parameters that should be set for the Pig job.

      Ex::
         return { 'YOUR_PARAM_NAME':'Your param value' }
      """

      return {}

   def pig_options(self):
      """
      Define the options that will be passed all the subclasses of PigTaskConfiguration
      """

      # Set the path to the log4j conf file
      options = ['-log4jconf', self._log4j_conf()]

      # Run the job locall if the run-locally option is set
      if self.run_locally:
         options.extend(['-x', 'local'])

      return options

   def pig_script_path(self):
      """
      Return the absolute path to the pig script to run
      """

      return "%s/pigscripts/%s" % (self.project_root, self.script())

   def script(self):
      """
      Return the name of the Pig script to run
      """

      raise NotImplementedError("subclass should define script")

   def output(self):
      """
      Return the output path of the script
      """

      return self.script_output()

   def script_output(self):
      """

      List of output paths generated by the pigscript

      Ex::
         return ['output_path']
      """

      raise NotImplementedError("subclass should define the script output path")

   def books_file(self):
      """
      Return 'aws_access_key_id' from client.cfg
      """
      return "%s/%s" % (self.input_root_path, 'books/MobyDick.txt')

   def create_s3_target(self, task_str):
      return create_task_s3_target(self.output_root_path, task_str)

   def create_local_target(self, task_str):
      return create_task_local_target(self.output_root_path, task_str)

   def _get_aws_access_key(self):
      """
      Return 'aws_access_key_id' from client.cfg
      """

      return configuration.get_config().get('aws', 'aws_access_key_id')

   def _get_aws_secret_access_key(self):
      """
      Return 'aws_secret_access_key' from client.cfg
      """

      return configuration.get_config().get('aws', 'aws_secret_access_key')

   def _log4j_conf(self):
      """
      Return the logging properties for log4j
      """

      return '%s/conf/log4j-cli-local-dev.properties' % (self.project_root);

   def _logfile(self):
      """
      The the location to store the pig logs
      """

      return '%s/logs/local-pig.log' % (self.project_root)

   def _get_root_path(self):
      """
      Return the root path of the project
      """

      if not self.root_path:
         self.root_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))

      return self.root_path

   def _s3_safe(self, s):
      """
      Ensure the S3 path specified is syntatically correct
      """
      return re.sub("[^0-9a-zA-Z]", '-', s)

class WordCount(PigscriptTask):

   def requires(self):
      """
      The requires method is how you build your dependency graph in Luigi.  Luigi will not
      run this task until all tasks returning in this list are complete.

      S3PathTask is a simple Luigi task that ensures some path already exists. As
      GenerateUserSignals is the first task in this Luigi pipeline it only requires that its
      expected input data can be found.
      """
      if self.run_locally:
         return [LocalInputFile(self.books_file())]
      else:
         return [S3PathTask(self.books_file())]

   def script_output(self):
      """
      The script_output method defines where output from the task will be stored.

      Luigi will check this output location before starting any tasks that depend on this task.
      """

      if self.run_locally:
         return [self.create_local_target('wordcount')]
      else:
         return [self.create_s3_target('wordcount')]

   def parameters(self):
      """
      This method defines the parameters that will be passed to the pig script when starting
      this pigscript.
      """

      return {
         'OUTPUT_ROOT_PATH': self.output_root_path,
         'INPUT_ROOT_PATH' : self.input_root_path,
      }

   def script(self):
      """
      The name of the pigscript to run
      """
      return "wordcount.pig"

class CreateEmrCluster(InitializeEmrCluster):
    """
    This task creates an EMR cluster for runnig the rest of the tasks on
    """

    output_root_path = luigi.Parameter()

    def output_token(self):
        return create_task_s3_target(self.output_root_path,
                                     self.__class__.__name__)

class RemoteWordCount(EmrPigTask):

   # The root path to where input data is located
   input_root_path = luigi.Parameter()

   # The root path to where output data will be written, a local or S3 path
   output_root_path = luigi.Parameter()

   def pig_args(self):
      args = []
      args.extend(['-p','INPUT_ROOT_PATH=%s' % self.input_root_path])
      args.extend(['-p','OUTPUT_ROOT_PATH=%s' % self.output_root_path])
      return args

   def output_token(self):

      return create_task_s3_target(self.output_root_path,
                                   self.__class__.__name__)

class ShutdownEmrCluster(TerminateEmrCluster):
    """
    This task terminates an EMR cluster
    """

    output_root_path = luigi.Parameter()

    def output_token(self):
        return create_task_s3_target(self.output_root_path,
                                     self.__class__.__name__)

if __name__ == "__main__":
   """
   We tell Luigi to run the last task in the task dependency graph.  Luigi will then
   work backwards to find any tasks with its requirements met and start from there.

   The first time this pipeline is run the only tasks with the requirements met will be
   the GenerateUserSignals and GenerateItemSignals tasks which will find their
   required input files in S3.
   """
   # luigi.run(main_task_cls=WordCount)
   #luigi.run(main_task_cls=CreateEmrCluster)
   # luigi.run(main_task_cls=RemoteWordCount)
   luigi.run(main_task_cls=ShutdownEmrCluster)
<|endoftext|>


<commit_before>#!/usr/bin/python
"""
This is a script to submit rabit job using hadoop streaming
submit the rabit process as mappers of MapReduce
"""
import argparse
import sys
import os
import time
import subprocess
import rabit_tracker as tracker

#!!! Set path to hadoop and hadoop streaming jar here
hadoop_binary = 'hadoop'
hadoop_streaming_jar = None

# code
hadoop_home = os.getenv('HADOOP_HOME')
if hadoop_home != None:
    if hadoop_binary == None:
        hadoop_binary = hadoop_home + '/bin/hadoop'
        assert os.path.exists(hadoop_binary), "HADDOP_HOME does not contain the hadoop binary"
    if hadoop_streaming_jar == None:
        hadoop_streaming_jar = hadoop_home + '/lib/hadoop-streaming.jar'
        assert os.path.exists(hadoop_streaming_jar), "HADDOP_HOME does not contain the haddop streaming jar"

if hadoop_binary == None or hadoop_streaming_jar == None:
    print 'Warning: Cannot auto-detect path to hadoop and hadoop-streaming jar, need to set them via arguments -hs and -hb'
    print '\tTo enable auto-detection, you can set enviroment variable HADOOP_HOME or modify rabit_hadoop.py line 14'

parser = argparse.ArgumentParser(description='Rabit script to submit rabit jobs using Hadoop Streaming')
parser.add_argument('-n', '--nworker', required=True, type=int,
                    help = 'number of worker proccess to be launched')
parser.add_argument('-i', '--input', required=True,
                    help = 'input path in HDFS')
parser.add_argument('-o', '--output', required=True,
                    help = 'output path in HDFS')
parser.add_argument('-v', '--verbose', default=0, choices=[0, 1], type=int,
                    help = 'print more messages into the console')
parser.add_argument('-ac', '--auto_file_cache', default=1, choices=[0, 1], type=int,
                    help = 'whether automatically cache the files in the command to hadoop localfile, this is on by default')
parser.add_argument('-f', '--files', nargs = '*',
                    help = 'the cached file list in mapreduce,'\
                        ' the submission script will automatically cache all the files which appears in command to local folder'\
                        ' This will also cause rewritten of all the file names in the command to current path,'\
                        ' for example `../../kmeans ../kmeans.conf` will be rewritten to `./kmeans kmeans.conf`'\
                        ' because the two files are cached to running folder.'\
                        ' You may need this option to cache additional files.'\
                        ' You can also use it to manually cache files when auto_file_cache is off')
parser.add_argument('--jobname', help = 'customize jobname in tracker')
parser.add_argument('--timeout', default=600000000, type=int,
                    help = 'timeout of each mapper job, automatically set to a very long time,'\
                        'normally you do not need to set this ')
parser.add_argument('-m', '--memory_mb', default=-1, type=int,
                    help = 'maximum memory used by the process, Guide: set it large (near mapred.cluster.max.map.memory.mb)'\
                        'if you are running multi-threading rabit,'\
                        'so that each node can occupy all the mapper slots in a machine for maximum performance')
if hadoop_binary == None:
    parser.add_argument('-hb', '--hadoop_binary', required = True,
                        help="path-to-hadoop binary folder")
else:
    parser.add_argument('-hb', '--hadoop_binary', default = hadoop_binary,
                        help="path-to-hadoop binary folder")

if hadoop_streaming_jar == None:
    parser.add_argument('-hs', '--hadoop_streaming_jar', required = True,
                        help='path-to hadoop streamimg jar file')
else:
    parser.add_argument('-hs', '--hadoop_streaming_jar', default = hadoop_streaming_jar,
                        help='path-to hadoop streamimg jar file')
parser.add_argument('command', nargs='+',
                    help = 'command for rabit program')
args = parser.parse_args()

if args.jobname is None:
    args.jobname = ('Rabit(nworker=%d):' % args.nworker) + args.command[0].split('/')[-1];

def hadoop_streaming(nworker, worker_args):
    fset = set()
    if args.auto_file_cache:
        for i in range(len(args.command)):
            f = args.command[i]
            if os.path.exists(f):
                fset.add(f)
                if i == 0:
                    args.command[i] = './' + args.command[i].split('/')[-1]
                else:
                    args.command[i] = args.command[i].split('/')[-1]
    cmd = '%s jar %s -D mapred.map.tasks=%d' % (args.hadoop_binary, args.hadoop_streaming_jar, nworker)
    cmd += ' -Dmapred.job.name=%s' % (args.jobname)
    cmd += ' -Dmapred.task.timeout=%d' % (args.timeout)
    cmd += ' -Dmapred.job.map.memory.mb=%d' % (args.memory_mb)
    cmd += ' -input %s -output %s' % (args.input, args.output)
    cmd += ' -mapper \"%s\" -reducer \"/bin/cat\" ' % (' '.join(args.command + worker_args))
    if args.files != None:
        for flst in args.files:
            for f in flst.split('#'):
                fset.add(f)
    for f in fset:
        cmd += ' -file %s' % f
    print cmd
    subprocess.check_call(cmd, shell = True)

tracker.submit(args.nworker, [], fun_submit = hadoop_streaming, verbose = args.verbose)
<commit_msg>auto jobname<commit_after>#!/usr/bin/python
"""
This is a script to submit rabit job using hadoop streaming
submit the rabit process as mappers of MapReduce
"""
import argparse
import sys
import os
import time
import subprocess
import rabit_tracker as tracker

#!!! Set path to hadoop and hadoop streaming jar here
hadoop_binary = 'hadoop'
hadoop_streaming_jar = None

# code
hadoop_home = os.getenv('HADOOP_HOME')
if hadoop_home != None:
    if hadoop_binary == None:
        hadoop_binary = hadoop_home + '/bin/hadoop'
        assert os.path.exists(hadoop_binary), "HADDOP_HOME does not contain the hadoop binary"
    if hadoop_streaming_jar == None:
        hadoop_streaming_jar = hadoop_home + '/lib/hadoop-streaming.jar'
        assert os.path.exists(hadoop_streaming_jar), "HADDOP_HOME does not contain the haddop streaming jar"

if hadoop_binary == None or hadoop_streaming_jar == None:
    print 'Warning: Cannot auto-detect path to hadoop and hadoop-streaming jar, need to set them via arguments -hs and -hb'
    print '\tTo enable auto-detection, you can set enviroment variable HADOOP_HOME or modify rabit_hadoop.py line 14'

parser = argparse.ArgumentParser(description='Rabit script to submit rabit jobs using Hadoop Streaming')
parser.add_argument('-n', '--nworker', required=True, type=int,
                    help = 'number of worker proccess to be launched')
parser.add_argument('-i', '--input', required=True,
                    help = 'input path in HDFS')
parser.add_argument('-o', '--output', required=True,
                    help = 'output path in HDFS')
parser.add_argument('-v', '--verbose', default=0, choices=[0, 1], type=int,
                    help = 'print more messages into the console')
parser.add_argument('-ac', '--auto_file_cache', default=1, choices=[0, 1], type=int,
                    help = 'whether automatically cache the files in the command to hadoop localfile, this is on by default')
parser.add_argument('-f', '--files', nargs = '*',
                    help = 'the cached file list in mapreduce,'\
                        ' the submission script will automatically cache all the files which appears in command to local folder'\
                        ' This will also cause rewritten of all the file names in the command to current path,'\
                        ' for example `../../kmeans ../kmeans.conf` will be rewritten to `./kmeans kmeans.conf`'\
                        ' because the two files are cached to running folder.'\
                        ' You may need this option to cache additional files.'\
                        ' You can also use it to manually cache files when auto_file_cache is off')
parser.add_argument('--jobname', default='auto', help = 'customize jobname in tracker')
parser.add_argument('--timeout', default=600000000, type=int,
                    help = 'timeout of each mapper job, automatically set to a very long time,'\
                        'normally you do not need to set this ')
parser.add_argument('-m', '--memory_mb', default=-1, type=int,
                    help = 'maximum memory used by the process, Guide: set it large (near mapred.cluster.max.map.memory.mb)'\
                        'if you are running multi-threading rabit,'\
                        'so that each node can occupy all the mapper slots in a machine for maximum performance')
if hadoop_binary == None:
    parser.add_argument('-hb', '--hadoop_binary', required = True,
                        help="path-to-hadoop binary folder")
else:
    parser.add_argument('-hb', '--hadoop_binary', default = hadoop_binary,
                        help="path-to-hadoop binary folder")

if hadoop_streaming_jar == None:
    parser.add_argument('-hs', '--hadoop_streaming_jar', required = True,
                        help='path-to hadoop streamimg jar file')
else:
    parser.add_argument('-hs', '--hadoop_streaming_jar', default = hadoop_streaming_jar,
                        help='path-to hadoop streamimg jar file')
parser.add_argument('command', nargs='+',
                    help = 'command for rabit program')
args = parser.parse_args()

if args.jobname == 'auto':
    args.jobname = ('Rabit(nworker=%d):' % args.nworker) + args.command[0].split('/')[-1];

def hadoop_streaming(nworker, worker_args):
    fset = set()
    if args.auto_file_cache:
        for i in range(len(args.command)):
            f = args.command[i]
            if os.path.exists(f):
                fset.add(f)
                if i == 0:
                    args.command[i] = './' + args.command[i].split('/')[-1]
                else:
                    args.command[i] = args.command[i].split('/')[-1]
    cmd = '%s jar %s -D mapred.map.tasks=%d' % (args.hadoop_binary, args.hadoop_streaming_jar, nworker)
    cmd += ' -Dmapred.job.name=%s' % (args.jobname)
    cmd += ' -Dmapred.task.timeout=%d' % (args.timeout)
    cmd += ' -Dmapred.job.map.memory.mb=%d' % (args.memory_mb)
    cmd += ' -input %s -output %s' % (args.input, args.output)
    cmd += ' -mapper \"%s\" -reducer \"/bin/cat\" ' % (' '.join(args.command + worker_args))
    if args.files != None:
        for flst in args.files:
            for f in flst.split('#'):
                fset.add(f)
    for f in fset:
        cmd += ' -file %s' % f
    print cmd
    subprocess.check_call(cmd, shell = True)

tracker.submit(args.nworker, [], fun_submit = hadoop_streaming, verbose = args.verbose)
<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals

import cgi
from collections import defaultdict, OrderedDict
import datetime
from decimal import Decimal
import json
import operator

from dash.utils import datetime_to_ms

from django.utils.safestring import mark_safe

from .models import Answer, Question, Response

class ChartJsonEncoder(json.JSONEncoder):
    """Encode millisecond timestamps & Decimal objects as floats."""

    def default(self, obj):
        if isinstance(obj, datetime.datetime):
            return datetime_to_ms(obj)
        elif isinstance(obj, Decimal):
            return float(obj)
        return json.JSONEncoder.default(self, obj)

def single_pollrun(pollrun, question, regions):
    """Chart data for a single pollrun.

    Will be a word cloud for open-ended questions, and pie chart of categories
    for everything else.
    """
    if question.type == Question.TYPE_OPEN:
        word_counts = pollrun.get_answer_word_counts(question, regions)
        chart_type = 'word'
        chart_data = word_cloud_data(word_counts)
    elif question.type in (Question.TYPE_MULTIPLE_CHOICE, Question.TYPE_KEYPAD, Question.TYPE_MENU):
        category_counts = pollrun.get_answer_category_counts(question, regions)
        chart_type = 'pie'
        chart_data = pie_chart_data(category_counts)
    elif question.type == Question.TYPE_NUMERIC:
        range_counts = pollrun.get_answer_auto_range_counts(question, regions)
        chart_type = 'column'
        chart_data = column_chart_data(range_counts)
    else:
        chart_type = None
        chart_data = []

    return chart_type, render_data(chart_data)

def multiple_pollruns_old(pollruns, question, regions):
    """Chart data for multiple pollruns of a poll."""

    if question.type == Question.TYPE_OPEN:
        overall_counts = defaultdict(int)

        for pollrun in pollruns:
            word_counts = pollrun.get_answer_word_counts(question, regions)
            for word, count in word_counts:
                overall_counts[word] += count

        sorted_counts = sorted(
            overall_counts.items(), key=operator.itemgetter(1), reverse=True)
        chart_type = 'word'
        chart_data = word_cloud_data(sorted_counts[:50])
    elif question.type == Question.TYPE_MULTIPLE_CHOICE:
        categories = set()
        counts_by_pollrun = OrderedDict()

        # fetch category counts for all pollruns, keeping track of all found
        # categories
        for pollrun in pollruns:
            category_counts = pollrun.get_answer_category_counts(question, regions)
            as_dict = dict(category_counts)
            counts_by_pollrun[pollrun] = as_dict

            for category in as_dict.keys():
                categories.add(category)

        categories = list(categories)
        category_series = defaultdict(list)

        for pollrun, category_counts in counts_by_pollrun.iteritems():
            for category in categories:
                count = category_counts.get(category, 0)
                category_series[category].append((pollrun.conducted_on, count))

        chart_type = 'time-area'
        chart_data = [{'name': cgi.escape(category), 'data': data}
                      for category, data in category_series.iteritems()]
    elif question.type == Question.TYPE_NUMERIC:
        chart_type = 'time-line'
        chart_data = []
        for pollrun in pollruns:
            average = pollrun.get_answer_numeric_average(question, regions)
            chart_data.append((pollrun.conducted_on, average))
    else:
        chart_type = None
        chart_data = []

    return chart_type, render_data(chart_data)

def multiple_pollruns(pollruns, question, regions):
    """Chart data for multiple pollruns of a poll."""
    # import ipdb; ipdb.set_trace()
    responses = Response.objects.filter(pollrun__in=pollruns)
    answers = Answer.objects.filter(response__in=responses, question=question)
    answers = answers.order_by('response__created_on')
    answer_sum_list, answer_average_list, date_list = answers.numeric_group_by_date()

    return answer_sum_list, answer_average_list, date_list

def word_cloud_data(word_counts):
    return [{'text': word, 'weight': count} for word, count in word_counts]

def pie_chart_data(category_counts):
    return [[cgi.escape(category), count] for category, count in category_counts]

def column_chart_data(range_counts):
    # highcharts needs the category labels and values separate for column charts
    if range_counts:
        labels, counts = zip(*range_counts)
        return [cgi.escape(l) for l in labels], counts
    else:
        return []

def render_data(chart_data):
    return mark_safe(json.dumps(chart_data, cls=ChartJsonEncoder))
<commit_msg>WIP response rate<commit_after>from __future__ import absolute_import, unicode_literals

import cgi
from collections import defaultdict, OrderedDict
import datetime
from decimal import Decimal
import json
import operator

from dash.utils import datetime_to_ms

from django.utils.safestring import mark_safe

from .models import Answer, Question, Response

class ChartJsonEncoder(json.JSONEncoder):
    """Encode millisecond timestamps & Decimal objects as floats."""

    def default(self, obj):
        if isinstance(obj, datetime.datetime):
            return datetime_to_ms(obj)
        elif isinstance(obj, Decimal):
            return float(obj)
        return json.JSONEncoder.default(self, obj)

def single_pollrun(pollrun, question, regions):
    """Chart data for a single pollrun.

    Will be a word cloud for open-ended questions, and pie chart of categories
    for everything else.
    """
    if question.type == Question.TYPE_OPEN:
        word_counts = pollrun.get_answer_word_counts(question, regions)
        chart_type = 'word'
        chart_data = word_cloud_data(word_counts)
    elif question.type in (Question.TYPE_MULTIPLE_CHOICE, Question.TYPE_KEYPAD, Question.TYPE_MENU):
        category_counts = pollrun.get_answer_category_counts(question, regions)
        chart_type = 'pie'
        chart_data = pie_chart_data(category_counts)
    elif question.type == Question.TYPE_NUMERIC:
        range_counts = pollrun.get_answer_auto_range_counts(question, regions)
        chart_type = 'column'
        chart_data = column_chart_data(range_counts)
    else:
        chart_type = None
        chart_data = []

    return chart_type, render_data(chart_data)

def multiple_pollruns_old(pollruns, question, regions):
    """Chart data for multiple pollruns of a poll."""

    if question.type == Question.TYPE_OPEN:
        overall_counts = defaultdict(int)

        for pollrun in pollruns:
            word_counts = pollrun.get_answer_word_counts(question, regions)
            for word, count in word_counts:
                overall_counts[word] += count

        sorted_counts = sorted(
            overall_counts.items(), key=operator.itemgetter(1), reverse=True)
        chart_type = 'word'
        chart_data = word_cloud_data(sorted_counts[:50])
    elif question.type == Question.TYPE_MULTIPLE_CHOICE:
        categories = set()
        counts_by_pollrun = OrderedDict()

        # fetch category counts for all pollruns, keeping track of all found
        # categories
        for pollrun in pollruns:
            category_counts = pollrun.get_answer_category_counts(question, regions)
            as_dict = dict(category_counts)
            counts_by_pollrun[pollrun] = as_dict

            for category in as_dict.keys():
                categories.add(category)

        categories = list(categories)
        category_series = defaultdict(list)

        for pollrun, category_counts in counts_by_pollrun.iteritems():
            for category in categories:
                count = category_counts.get(category, 0)
                category_series[category].append((pollrun.conducted_on, count))

        chart_type = 'time-area'
        chart_data = [{'name': cgi.escape(category), 'data': data}
                      for category, data in category_series.iteritems()]
    elif question.type == Question.TYPE_NUMERIC:
        chart_type = 'time-line'
        chart_data = []
        for pollrun in pollruns:
            average = pollrun.get_answer_numeric_average(question, regions)
            chart_data.append((pollrun.conducted_on, average))
    else:
        chart_type = None
        chart_data = []

    return chart_type, render_data(chart_data)

def multiple_pollruns(pollruns, question, regions):
    """Chart data for multiple pollruns of a poll."""
    # import ipdb; ipdb.set_trace()
    responses = Response.objects.filter(pollrun__in=pollruns)
    # import ipdb; ipdb.set_trace()
    answers = Answer.objects.filter(response__in=responses, question=question)
    answers = answers.order_by('response__created_on')
    answer_sum_list, answer_average_list, date_list = answers.numeric_group_by_date()

    # TODO response rate
    # res2 = responses.filter(status='E')
    # res2 = res2.extra({'date_created':"date(created_on)"})
    # res2 = res2.values('date_created').annotate(created_count=Count('id'))

    return answer_sum_list, answer_average_list, date_list

def word_cloud_data(word_counts):
    return [{'text': word, 'weight': count} for word, count in word_counts]

def pie_chart_data(category_counts):
    return [[cgi.escape(category), count] for category, count in category_counts]

def column_chart_data(range_counts):
    # highcharts needs the category labels and values separate for column charts
    if range_counts:
        labels, counts = zip(*range_counts)
        return [cgi.escape(l) for l in labels], counts
    else:
        return []

def render_data(chart_data):
    return mark_safe(json.dumps(chart_data, cls=ChartJsonEncoder))
<|endoftext|>