<commit_before>from django.http import HttpResponse, HttpResponseRedirect
from restclients.dao import SWS_DAO, PWS_DAO, GWS_DAO
from restclients.gws import GWS
from django.conf import settings
import json
import re

def proxy(request, service, url):

    if not hasattr(settings, "RESTCLIENTS_ADMIN_GROUP"):
        print "You must have a group in GWS defined as your admin group."
        print 'Configure that using RESTCLIENTS_ADMIN_GROUP="u_foo_bar"'
        raise Exception("Missing RESTCLIENTS_ADMIN_GROUP in settings")

    gws = GWS()
    members = gws.get_effective_members(settings.MYUW_ADMIN_GROUP)

    if settings.DEBUG:
        actual_user = 'javerage'
    else:
        actual_user = request.user.username

    is_admin = False

    # XXX - use a new GWS method for is effective member :(
    for member in members:
        if member.uwnetid == actual_user:
            is_admin = True
            break

    if is_admin == False:
        return HttpResponseRedirect("/")

    if service == "sws":
        dao = SWS_DAO()
    elif service == "pws":
        dao = PWS_DAO()
    elif service == "gws":
        dao = GWS_DAO()
    else:
        raise Exception("Unknown service: %s" % service)

    url = "/%s" % url

    response = dao.getURL(url, {})
    content = response.data

    # Assume json, and try to format it.
    try:
        content = format_json(service, content)
    except Exception as e:
        content = format_html(content)

    content = add_response_info(response, content)

    return HttpResponse(content)

def add_response_info(response, content):
    meta = []
    meta.append("Code: %s" % response.status)

    for header in response.headers:
        meta.append("%s: %s" % (header, response.headers[header]))

    return "
".join(["
".join(meta), "Response Body:", content])

def format_json(service, content):
    json_data = json.loads(content)
    formatted = json.dumps(json_data, sort_keys=True, indent=4)
    formatted = formatted.replace("&", "&")
    formatted = formatted.replace("<", "<")
    formatted = formatted.replace(">", ">")
    formatted = formatted.replace(" ", " ")
    formatted = formatted.replace("\n", "
\n")

    formatted = re.sub(r"\"/(.*?)\"", r"/\1" % service, formatted)

    return formatted

def format_html(content):
    return content

<commit_msg>Make links in html representations point to the right place<commit_after>from django.http import HttpResponse, HttpResponseRedirect
from restclients.dao import SWS_DAO, PWS_DAO, GWS_DAO
from restclients.gws import GWS
from django.conf import settings
import json
import re

def proxy(request, service, url):

    if not hasattr(settings, "RESTCLIENTS_ADMIN_GROUP"):
        print "You must have a group in GWS defined as your admin group."
        print 'Configure that using RESTCLIENTS_ADMIN_GROUP="u_foo_bar"'
        raise Exception("Missing RESTCLIENTS_ADMIN_GROUP in settings")

    gws = GWS()
    members = gws.get_effective_members(settings.MYUW_ADMIN_GROUP)

    if settings.DEBUG:
        actual_user = 'javerage'
    else:
        actual_user = request.user.username

    is_admin = False

    # XXX - use a new GWS method for is effective member :(
    for member in members:
        if member.uwnetid == actual_user:
            is_admin = True
            break

    if is_admin == False:
        return HttpResponseRedirect("/")

    if service == "sws":
        dao = SWS_DAO()
    elif service == "pws":
        dao = PWS_DAO()
    elif service == "gws":
        dao = GWS_DAO()
    else:
        raise Exception("Unknown service: %s" % service)

    url = "/%s" % url

    response = dao.getURL(url, {})
    content = response.data

    # Assume json, and try to format it.
    try:
        content = format_json(service, content)
    except Exception as e:
        content = format_html(service, content)

    content = add_response_info(response, content)

    return HttpResponse(content)

def add_response_info(response, content):
    meta = []
    meta.append("Code: %s" % response.status)

    for header in response.headers:
        meta.append("%s: %s" % (header, response.headers[header]))

    return "
".join(["
".join(meta), "Response Body:", content])

def format_json(service, content):
    json_data = json.loads(content)
    formatted = json.dumps(json_data, sort_keys=True, indent=4)
    formatted = formatted.replace("&", "&")
    formatted = formatted.replace("<", "<")
    formatted = formatted.replace(">", ">")
    formatted = formatted.replace(" ", " ")
    formatted = formatted.replace("\n", "
\n")

    formatted = re.sub(r"\"/(.*?)\"", r"/\1" % service, formatted)

    return formatted

def format_html(service, content):
    formatted = re.sub(r"href\s*=\s*\"/(.*?)\"", r"href='/restclients/view/%s/\1'" % service, content)
    return formatted

<|endoftext|>


<commit_before><commit_msg>iterate over sample sizes<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: ai ts=4 sts=4 et sw=4 nu

from __future__ import (unicode_literals, absolute_import,
                        division, print_function)
import logging
import os

from django.core.management.base import BaseCommand
from optparse import make_option
from py3compat import PY2

from snisi_core.models.Entities import (
    Entity, HealthEntity, AdministrativeEntity, EntityType)

if PY2:
    import unicodecsv as csv
else:
    import csv

logger = logging.getLogger(__name__)

class Command(BaseCommand):

    option_list = BaseCommand.option_list + (
        make_option('-f',
                    help='CSV file',
                    action='store',
                    dest='filename'),
    )

    def handle(self, *args, **options):

        if not os.path.exists(options.get('filename') or ""):
            logger.error("CSV file `{}` does not exist."
                         .format(options.get('filename')))
            return

        headers = ['action', 'slug', 'parent', 'name', 'type',
                   'has_urenam', 'has_urenas', 'has_ureni', 'main_entity']
        input_csv_file = open(options.get('filename'), 'r')
        csv_reader = csv.DictReader(input_csv_file, fieldnames=headers)

        has_matrix = {
            'y': True,
            'n': False
        }

        for entry in csv_reader:
            if csv_reader.line_num == 1:
                continue
            logger.debug(entry)

            if entry.get('action') == 'create':
                logger.info("Creating {}".format(entry.get('name')))
                parent = Entity.get_or_none(entry.get('parent'))
                etype = EntityType.get_or_none(entry.get('type'))
                hcls = HealthEntity if etype.slug.startswith('health_') \
                    else AdministrativeEntity
                entity = hcls.objects.create(
                    slug=entry.get('slug').upper(),
                    name=entry.get('name').upper(),
                    type=etype,
                    parent=parent)
                if entry.get('has_urenam') == 'y':
                    entity.has_urenam = True
                if entry.get('has_urenas') == 'y':
                    entity.has_urenas = True
                if entry.get('has_ureni') == 'y':
                    entity.has_ureni = True
                entity.save()
            elif entry.get('action') == 'disable':
                entity = Entity.get_or_none(entry.get('slug'))
                logger.info("Disabling {}".format(entity))
                entity.active = False
                entity.save()
            elif entry.get('action') == 'update':
                entity = Entity.get_or_none(entry.get('slug'))
                logger.info("Updating {}".format(entity))
                if entry.get('name'):
                    entity.name = entry.get('name').upper()
                    entity.save()
                if entry.get('type'):
                    etype = EntityType.get_or_none(entry.get('type'))
                    entity.type = etype
                    entity.save()
                if entry.get('parent'):
                    parent = Entity.get_or_none(entry.get('parent'))
                    entity.parent = parent
                    entity.save()
                # update main_entity ?
                if entry.get('main_entity'):
                    main_entity = Entity.get_or_none(entry.get('main_entity'))
                    entity.main_entity = main_entity
                    entity.save()
                if entry.get('has_urenam'):
                    entity.has_urenam = has_matrix.get(
                        entry.get('has_urenam').lower(), False)
                    entity.save()
                if entry.get('has_urenas'):
                    entity.has_urenas = has_matrix.get(
                        entry.get('has_urenas').lower(), False)
                    entity.save()
                if entry.get('has_ureni'):
                    entity.has_ureni = has_matrix.get(
                        entry.get('has_ureni').lower(), False)
                    entity.save()

        logger.info("Done")
<commit_msg>Added enable action<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: ai ts=4 sts=4 et sw=4 nu

from __future__ import (unicode_literals, absolute_import,
                        division, print_function)
import logging
import os

from django.core.management.base import BaseCommand
from optparse import make_option
from py3compat import PY2

from snisi_core.models.Entities import (
    Entity, HealthEntity, AdministrativeEntity, EntityType)

if PY2:
    import unicodecsv as csv
else:
    import csv

logger = logging.getLogger(__name__)

class Command(BaseCommand):

    option_list = BaseCommand.option_list + (
        make_option('-f',
                    help='CSV file',
                    action='store',
                    dest='filename'),
    )

    def handle(self, *args, **options):

        if not os.path.exists(options.get('filename') or ""):
            logger.error("CSV file `{}` does not exist."
                         .format(options.get('filename')))
            return

        headers = ['action', 'slug', 'parent', 'name', 'type',
                   'has_urenam', 'has_urenas', 'has_ureni', 'main_entity']
        input_csv_file = open(options.get('filename'), 'r')
        csv_reader = csv.DictReader(input_csv_file, fieldnames=headers)

        has_matrix = {
            'y': True,
            'n': False
        }

        for entry in csv_reader:
            if csv_reader.line_num == 1:
                continue
            logger.debug(entry)

            if entry.get('action') == 'create':
                logger.info("Creating {}".format(entry.get('name')))
                parent = Entity.get_or_none(entry.get('parent'))
                etype = EntityType.get_or_none(entry.get('type'))
                hcls = HealthEntity if etype.slug.startswith('health_') \
                    else AdministrativeEntity
                entity = hcls.objects.create(
                    slug=entry.get('slug').upper(),
                    name=entry.get('name').upper(),
                    type=etype,
                    parent=parent)
                if entry.get('has_urenam') == 'y':
                    entity.has_urenam = True
                if entry.get('has_urenas') == 'y':
                    entity.has_urenas = True
                if entry.get('has_ureni') == 'y':
                    entity.has_ureni = True
                entity.save()
            elif entry.get('action') == 'disable':
                entity = Entity.get_or_none(entry.get('slug'))
                logger.info("Disabling {}".format(entity))
                entity.active = False
                entity.save()
            elif entry.get('action') == 'enable':
                entity = Entity.get_or_none(entry.get('slug'))
                logger.info("Enabling {}".format(entity))
                entity.active = True
                entity.save()
            elif entry.get('action') == 'update':
                entity = Entity.get_or_none(entry.get('slug'))
                logger.info("Updating {}".format(entity))
                if entry.get('name'):
                    entity.name = entry.get('name').upper()
                    entity.save()
                if entry.get('type'):
                    etype = EntityType.get_or_none(entry.get('type'))
                    entity.type = etype
                    entity.save()
                if entry.get('parent'):
                    parent = Entity.get_or_none(entry.get('parent'))
                    entity.parent = parent
                    entity.save()
                # update main_entity ?
                if entry.get('main_entity'):
                    main_entity = Entity.get_or_none(entry.get('main_entity'))
                    entity.main_entity = main_entity
                    entity.save()
                if entry.get('has_urenam'):
                    entity.has_urenam = has_matrix.get(
                        entry.get('has_urenam').lower(), False)
                    entity.save()
                if entry.get('has_urenas'):
                    entity.has_urenas = has_matrix.get(
                        entry.get('has_urenas').lower(), False)
                    entity.save()
                if entry.get('has_ureni'):
                    entity.has_ureni = has_matrix.get(
                        entry.get('has_ureni').lower(), False)
                    entity.save()

        logger.info("Done")
<|endoftext|>


<commit_before>#
# Project:   retdec-python
# Copyright: (c) 2015-2016 by Petr Zemek <s3rvac@gmail.com> and contributors
# License:   MIT, see the LICENSE file for more details
#

"""Access to the decompiler (decompilation of files)."""

from retdec.decompilation import Decompilation
from retdec.exceptions import MissingParameterError
from retdec.file import File
from retdec.service import Service

class Decompiler(Service):
    """Access to the decompilation service."""

    def start_decompilation(self, **kwargs):
        """Starts a decompilation with the given parameters.

        :param input_file: File to be analyzed (**required**).
        :type input_file: str or file-like object
        :param pdb_file: A PDB file associated with `input_file` containing
            debugging information.
        :type pdb_file: str or file-like object
        :param mode: Decompilation mode.
        :type mode: str
        :param target_language: Target high-level language.
        :type target_language: str
        :param graph_format: Format of the generated call and control-flow
            graphs.
        :type graph_format: str
        :param decomp_var_names: Naming style for variables.
        :type decomp_var_names: str
        :param decomp_optimizations: Level of optimizations performed by the
            decompiler.
        :type decomp_optimizations: str
        :param decomp_unreach_funcs: Should all functions be decompiled, even
            if they are not reachable from the main function?
        :type decomp_unreach_funcs: bool
        :param decomp_emit_addresses: Should addresses in comments be emitted
            in the generated code?
        :type decomp_emit_addresses: bool
        :param architecture: Architecture. The precise meaning depends on the
            used `mode`.
        :type architecture: str
        :param file_format: File format. The precise meaning depends on the
            used `mode`.
        :type file_format: str
        :param comp_compiler: Compiler to be used when compiling input C source
            files.
        :type comp_compiler: str
        :param comp_optimizations: Compiler optimizations to be used when
            compiling input C source files.
        :type comp_optimizations: str
        :param comp_debug: Should the input C source file be compiled with
            debugging information?
        :type comp_debug: bool
        :param comp_strip: Should the compiled input C source file be stripped?
        :type comp_strip: bool
        :param sel_decomp_funcs: Decompile only the selected functions. It can
            be either an iterable of function names (e.g. ``['func1', 'func2']``) or
            a string with comma-separated function names (e.g. ``'func1,
            func2'``).
        :type sel_decomp_funcs: str/iterable
        :param sel_decomp_ranges: Decompile only the selected address ranges.
            It can be either an iterable of ranges (e.g. ``[(0x100, 0x200),
            (0x400, 0x500)]``) or a string with comma-separated ranges (e.g.
            ``'0x100-0x200,0x400-0x500'``).
        :type sel_decomp_ranges: str/iterable
        :param sel_decomp_decoding: What instructions should be decoded when
            either `sel_decomp_funcs` or `sel_decomp_ranges` is given?
        :type sel_decomp_decoding: str
        :param raw_entry_point: Specify the virtual memory address where execution
            flow should start in the raw machine code (``raw`` mode only).
        :type raw_entry_point: str
        :param raw_section_vma: Specify an address where the section should be created
            from the raw machine code (``raw`` mode only).
        :type raw_section_vma: str
        :param raw_endian: Indicate the endianness (``'little'`` or ``'big'``)
            of the machine code (``raw`` mode only).
        :type raw_endian: str
        :param generate_cg: Should a call graph be generated?
        :type generate_cg: bool
        :param generate_cfgs: Should control-flow graphs for all functions be
            generated?
        :type generate_cfgs: bool
        :param generate_archive: Should an archive containing all outputs from
            the decompilation be generated?
        :type generate_archive: bool

        :returns: Started decompilation
            (:class:`~retdec.decompilation.Decompilation`).

        If `mode` is not given, it is automatically determined based on the
        name of ``input_file``. If the input file ends with ``.c`` or ``.C``,
        the mode is set to ``c``. Otherwise, the mode is set to ``bin``.

        See the `official documentation
        <https://retdec.com/api/docs/decompiler.html#decompilation-parameters>`_
        for more information about the parameters.
        """
        conn = self._create_new_api_connection('/decompiler/decompilations')
        id = self._start_decompilation(conn, kwargs)
        return Decompilation(id, conn)

    def _start_decompilation(self, conn, kwargs):
        """Starts a decompilation with the given parameters.

        :param retdec.conn.APIConnection conn: Connection to the API to be used
            for sending API requests.
        :param dict kwargs: Parameters for the decompilation.

        :returns: Unique identifier of the decompilation.
        """
        files = {
            'input': self._get_input_file(kwargs)
        }
        self._add_pdb_file_when_given(files, kwargs)
        params = {
            'mode': self._get_mode_param(files['input'], kwargs)
        }
        self._add_param_when_given('target_language', params, kwargs)
        self._add_param_when_given('graph_format', params, kwargs)
        self._add_param_when_given('decomp_var_names', params, kwargs)
        self._add_param_when_given('decomp_optimizations', params, kwargs)
        self._add_param_when_given('decomp_unreach_funcs', params, kwargs)
        self._add_param_when_given('decomp_emit_addresses', params, kwargs)
        self._add_param_when_given('architecture', params, kwargs)
        self._add_param_when_given('file_format', params, kwargs)
        self._add_param_when_given('comp_compiler', params, kwargs)
        self._add_param_when_given('comp_debug', params, kwargs)
        self._add_param_when_given('comp_strip', params, kwargs)
        self._add_comp_optimizations_param_when_given(params, kwargs)
        self._add_sel_decomp_funcs_param_when_given(params, kwargs)
        self._add_sel_decomp_ranges_param_when_given(params, kwargs)
        self._add_param_when_given('sel_decomp_decoding', params, kwargs)
        self._add_param_when_given('raw_entry_point', params, kwargs)
        self._add_param_when_given('raw_section_vma', params, kwargs)
        self._add_param_when_given('raw_endian', params, kwargs)
        self._add_param_when_given('generate_archive', params, kwargs)
        self._add_param_when_given('generate_cg', params, kwargs)
        self._add_param_when_given('generate_cfgs', params, kwargs)
        response = conn.send_post_request(files=files, params=params)
        return response['id']

    def _get_input_file(self, kwargs):
        """Returns the input file to be decompiled."""
        try:
            return File(kwargs['input_file'])
        except KeyError:
            raise MissingParameterError('input_file')

    def _add_pdb_file_when_given(self, files, kwargs):
        """Adds a PDB file to `files` when it was given."""
        pdb_file = kwargs.get('pdb_file', None)
        if pdb_file is not None:
            files['pdb'] = File(pdb_file)

    def _get_mode_param(self, input_file, kwargs):
        """Returns a decompilation mode to be used."""
        return self._get_param(
            'mode',
            kwargs,
            choices={'c', 'bin', 'raw'},
            default=self._get_default_mode(input_file)
        )

    def _get_default_mode(self, input_file):
        """Returns a default decompilation mode to be used based on the given
        input file's name.
        """
        return 'c' if input_file.name.lower().endswith('.c') else 'bin'

    def _add_param_when_given(self, param, params, kwargs):
        """Adds `param` to `params` when given in `kwargs`."""
        value = kwargs.get(param, None)
        if value is not None:
            params[param] = value

    def _add_comp_optimizations_param_when_given(self, params, kwargs):
        """Adds the ``comp_optimizations`` parameter to `params` when given in
        `kwargs`.
        """
        value = kwargs.get('comp_optimizations', None)
        if value is not None:
            # The retdec.com API expects the optimization level to start with a
            # dash (e.g. "-O1" instead of just "O1"). To be user-friendly,
            # allow passing just "O1" and add the dash automatically.
            if not value.startswith('-'):
                value = '-' + value
            params['comp_optimizations'] = value

    def _add_sel_decomp_funcs_param_when_given(self, params, kwargs):
        """Adds the ``sel_decomp_funcs`` parameter to `params` when given in
        `kwargs`.
        """
        value = kwargs.get('sel_decomp_funcs', None)
        if value is not None:
            if not isinstance(value, str):
                value = ','.join(value)
            params['sel_decomp_funcs'] = value

    def _add_sel_decomp_ranges_param_when_given(self, params, kwargs):
        """Adds the ``sel_decomp_ranges`` parameter to `params` when given in
        `kwargs`.
        """
        def ranges2str(ranges):
            return ','.join(
                range2str(range) for range in ranges
            )

        def range2str(range):
            if isinstance(range, tuple):
                assert len(range) == 2, 'invalid range: {}'.format(range)
                return '{}-{}'.format(
                    address2str(range[0]), address2str(range[1])
                )
            return str(range)

        def address2str(address):
            if isinstance(address, int):
                return hex(address)
            return str(address)

        value = kwargs.get('sel_decomp_ranges', None)
        if value is not None:
            if not isinstance(value, str):
                value = ranges2str(value)
            params['sel_decomp_ranges'] = value

    def __repr__(self):
        return '<{} api_url={!r}>'.format(
            __name__ + '.' + self.__class__.__name__,
            self.api_url
        )
<commit_msg>Rephrase the description of raw-mode related parameters.<commit_after>#
# Project:   retdec-python
# Copyright: (c) 2015-2016 by Petr Zemek <s3rvac@gmail.com> and contributors
# License:   MIT, see the LICENSE file for more details
#

"""Access to the decompiler (decompilation of files)."""

from retdec.decompilation import Decompilation
from retdec.exceptions import MissingParameterError
from retdec.file import File
from retdec.service import Service

class Decompiler(Service):
    """Access to the decompilation service."""

    def start_decompilation(self, **kwargs):
        """Starts a decompilation with the given parameters.

        :param input_file: File to be analyzed (**required**).
        :type input_file: str or file-like object
        :param pdb_file: A PDB file associated with `input_file` containing
            debugging information.
        :type pdb_file: str or file-like object
        :param mode: Decompilation mode.
        :type mode: str
        :param target_language: Target high-level language.
        :type target_language: str
        :param graph_format: Format of the generated call and control-flow
            graphs.
        :type graph_format: str
        :param decomp_var_names: Naming style for variables.
        :type decomp_var_names: str
        :param decomp_optimizations: Level of optimizations performed by the
            decompiler.
        :type decomp_optimizations: str
        :param decomp_unreach_funcs: Should all functions be decompiled, even
            if they are not reachable from the main function?
        :type decomp_unreach_funcs: bool
        :param decomp_emit_addresses: Should addresses in comments be emitted
            in the generated code?
        :type decomp_emit_addresses: bool
        :param architecture: Architecture. The precise meaning depends on the
            used `mode`.
        :type architecture: str
        :param file_format: File format. The precise meaning depends on the
            used `mode`.
        :type file_format: str
        :param comp_compiler: Compiler to be used when compiling input C source
            files.
        :type comp_compiler: str
        :param comp_optimizations: Compiler optimizations to be used when
            compiling input C source files.
        :type comp_optimizations: str
        :param comp_debug: Should the input C source file be compiled with
            debugging information?
        :type comp_debug: bool
        :param comp_strip: Should the compiled input C source file be stripped?
        :type comp_strip: bool
        :param sel_decomp_funcs: Decompile only the selected functions. It can
            be either an iterable of function names (e.g. ``['func1', 'func2']``) or
            a string with comma-separated function names (e.g. ``'func1,
            func2'``).
        :type sel_decomp_funcs: str/iterable
        :param sel_decomp_ranges: Decompile only the selected address ranges.
            It can be either an iterable of ranges (e.g. ``[(0x100, 0x200),
            (0x400, 0x500)]``) or a string with comma-separated ranges (e.g.
            ``'0x100-0x200,0x400-0x500'``).
        :type sel_decomp_ranges: str/iterable
        :param sel_decomp_decoding: What instructions should be decoded when
            either `sel_decomp_funcs` or `sel_decomp_ranges` is given?
        :type sel_decomp_decoding: str
        :param raw_entry_point: Virtual memory address where execution
            flow should start in the raw machine code. Only for the ``raw``
            mode.
        :type raw_entry_point: str
        :param raw_section_vma: Address where the section created from the raw
            machine code will be placed in virtual memory. Only for the
            ``raw`` mode.
        :type raw_section_vma: str
        :param raw_endian: Endianness of the raw machine code (``'little'`` or
            ``'big'``). Only for the ``raw`` mode.
        :type raw_endian: str
        :param generate_cg: Should a call graph be generated?
        :type generate_cg: bool
        :param generate_cfgs: Should control-flow graphs for all functions be
            generated?
        :type generate_cfgs: bool
        :param generate_archive: Should an archive containing all outputs from
            the decompilation be generated?
        :type generate_archive: bool

        :returns: Started decompilation
            (:class:`~retdec.decompilation.Decompilation`).

        If `mode` is not given, it is automatically determined based on the
        name of ``input_file``. If the input file ends with ``.c`` or ``.C``,
        the mode is set to ``c``. Otherwise, the mode is set to ``bin``.

        See the `official documentation
        <https://retdec.com/api/docs/decompiler.html#decompilation-parameters>`_
        for more information about the parameters.
        """
        conn = self._create_new_api_connection('/decompiler/decompilations')
        id = self._start_decompilation(conn, kwargs)
        return Decompilation(id, conn)

    def _start_decompilation(self, conn, kwargs):
        """Starts a decompilation with the given parameters.

        :param retdec.conn.APIConnection conn: Connection to the API to be used
            for sending API requests.
        :param dict kwargs: Parameters for the decompilation.

        :returns: Unique identifier of the decompilation.
        """
        files = {
            'input': self._get_input_file(kwargs)
        }
        self._add_pdb_file_when_given(files, kwargs)
        params = {
            'mode': self._get_mode_param(files['input'], kwargs)
        }
        self._add_param_when_given('target_language', params, kwargs)
        self._add_param_when_given('graph_format', params, kwargs)
        self._add_param_when_given('decomp_var_names', params, kwargs)
        self._add_param_when_given('decomp_optimizations', params, kwargs)
        self._add_param_when_given('decomp_unreach_funcs', params, kwargs)
        self._add_param_when_given('decomp_emit_addresses', params, kwargs)
        self._add_param_when_given('architecture', params, kwargs)
        self._add_param_when_given('file_format', params, kwargs)
        self._add_param_when_given('comp_compiler', params, kwargs)
        self._add_param_when_given('comp_debug', params, kwargs)
        self._add_param_when_given('comp_strip', params, kwargs)
        self._add_comp_optimizations_param_when_given(params, kwargs)
        self._add_sel_decomp_funcs_param_when_given(params, kwargs)
        self._add_sel_decomp_ranges_param_when_given(params, kwargs)
        self._add_param_when_given('sel_decomp_decoding', params, kwargs)
        self._add_param_when_given('raw_entry_point', params, kwargs)
        self._add_param_when_given('raw_section_vma', params, kwargs)
        self._add_param_when_given('raw_endian', params, kwargs)
        self._add_param_when_given('generate_archive', params, kwargs)
        self._add_param_when_given('generate_cg', params, kwargs)
        self._add_param_when_given('generate_cfgs', params, kwargs)
        response = conn.send_post_request(files=files, params=params)
        return response['id']

    def _get_input_file(self, kwargs):
        """Returns the input file to be decompiled."""
        try:
            return File(kwargs['input_file'])
        except KeyError:
            raise MissingParameterError('input_file')

    def _add_pdb_file_when_given(self, files, kwargs):
        """Adds a PDB file to `files` when it was given."""
        pdb_file = kwargs.get('pdb_file', None)
        if pdb_file is not None:
            files['pdb'] = File(pdb_file)

    def _get_mode_param(self, input_file, kwargs):
        """Returns a decompilation mode to be used."""
        return self._get_param(
            'mode',
            kwargs,
            choices={'c', 'bin', 'raw'},
            default=self._get_default_mode(input_file)
        )

    def _get_default_mode(self, input_file):
        """Returns a default decompilation mode to be used based on the given
        input file's name.
        """
        return 'c' if input_file.name.lower().endswith('.c') else 'bin'

    def _add_param_when_given(self, param, params, kwargs):
        """Adds `param` to `params` when given in `kwargs`."""
        value = kwargs.get(param, None)
        if value is not None:
            params[param] = value

    def _add_comp_optimizations_param_when_given(self, params, kwargs):
        """Adds the ``comp_optimizations`` parameter to `params` when given in
        `kwargs`.
        """
        value = kwargs.get('comp_optimizations', None)
        if value is not None:
            # The retdec.com API expects the optimization level to start with a
            # dash (e.g. "-O1" instead of just "O1"). To be user-friendly,
            # allow passing just "O1" and add the dash automatically.
            if not value.startswith('-'):
                value = '-' + value
            params['comp_optimizations'] = value

    def _add_sel_decomp_funcs_param_when_given(self, params, kwargs):
        """Adds the ``sel_decomp_funcs`` parameter to `params` when given in
        `kwargs`.
        """
        value = kwargs.get('sel_decomp_funcs', None)
        if value is not None:
            if not isinstance(value, str):
                value = ','.join(value)
            params['sel_decomp_funcs'] = value

    def _add_sel_decomp_ranges_param_when_given(self, params, kwargs):
        """Adds the ``sel_decomp_ranges`` parameter to `params` when given in
        `kwargs`.
        """
        def ranges2str(ranges):
            return ','.join(
                range2str(range) for range in ranges
            )

        def range2str(range):
            if isinstance(range, tuple):
                assert len(range) == 2, 'invalid range: {}'.format(range)
                return '{}-{}'.format(
                    address2str(range[0]), address2str(range[1])
                )
            return str(range)

        def address2str(address):
            if isinstance(address, int):
                return hex(address)
            return str(address)

        value = kwargs.get('sel_decomp_ranges', None)
        if value is not None:
            if not isinstance(value, str):
                value = ranges2str(value)
            params['sel_decomp_ranges'] = value

    def __repr__(self):
        return '<{} api_url={!r}>'.format(
            __name__ + '.' + self.__class__.__name__,
            self.api_url
        )
<|endoftext|>


<commit_before>from django import template
from django.core.urlresolvers import reverse
from django.template import Context, Template
from django.utils.translation import ugettext_lazy as _
from product import active_product_types

register = template.Library()

def js_make_select_readonly(select):
    #This is really just a mini-template
    #select must be a jquery object
    return """
    select = %(select)s;
    value = select.attr("value");
    if (value){
        text = "";
        for (c in select.children()){
          if (select.children()[c].value == value){
            text = select.children()[c].innerHTML;
            break;
          }
        }
        select.before("" + text + "<input type=\\"hidden\\" name=\\"" + select.attr("name") + "\\" value=\\"" + value + "\\" \>");
        select.remove();
    }
    """ % {'select':select}

register.simple_tag(js_make_select_readonly)

def edit_subtypes(product):
    output = '
'
    subtypes = product.get_subtypes()
    for app, subtype in active_product_types():
        is_config = "ConfigurableProduct" in subtypes
        app_label = app.split(".")[-1]
        if subtype in subtypes:
            edit_url = reverse('admin:%s_%s_change' %
                               (app_label, subtype.lower()),
                                args=(product.pk,))
            output += ('
' % edit_url +
                       _('Edit %(subtype)s') % {'subtype': subtype} +
                       '
')
            if is_config or subtype=="ProductVariation":
                 output += '
Variation Manager
' % (reverse("satchmo_admin_variation_manager", args = [product.id]))
        else:
            if not(is_config and subtype=="ProductVariation"):
                add_url = reverse('admin:%s_%s_add' %
                                  (app_label, subtype.lower()))
                output += ('
' % (add_url, product.id) +
                           _('Add %(subtype)s') % {'subtype': subtype} +
                           '
')
    output += '
'
    return output

register.simple_tag(edit_subtypes)

def list_variations(configurableproduct):
    opts = configurableproduct.get_all_options()
    output = "{% load admin_modify adminmedia %}"
    output += "
"
    for p_opt in opts:
        opt_strs = []
        [opt_strs.append(opt.name) for opt in p_opt]
        opt_str = ', '.join(opt_strs)

        product = configurableproduct.get_product_from_options(p_opt)
        if product:
            p_url = '/admin/product/product/%s/' % product.pk
            pv_url = '/admin/product/productvariation/%s/delete/' % product.pk
            output += """

            %s
            %s
            %s

            """ % (opt_str, p_url, product.slug, pv_url,
                _("Delete ProductVariation"))
        else:
            #opt_pks = [str(opt.pk) for opt in p_opt]
            #opt_pks = ','.join(opt_pks)
            # TODO [NFA]: Blocked by Django ticket #7738.
            opt_pks = ''
            add_url = ('/admin/product/productvariation/add/' +
                "?product=%s&parent=%s&options=%s" % (
                configurableproduct.product.pk, configurableproduct.product.pk,
                opt_pks))
            output += """

            %s

              Add Variation

            """ % (opt_str, add_url)
    output += "
"
    t = Template(output)
    return t.render(Context())

register.simple_tag(list_variations)

<commit_msg>satchmo_product_admin: don't wrap in ()<commit_after>from django import template
from django.core.urlresolvers import reverse
from django.template import Context, Template
from django.utils.translation import ugettext_lazy as _
from product import active_product_types

register = template.Library()

def js_make_select_readonly(select):
    #This is really just a mini-template
    #select must be a jquery object
    return """
    select = %(select)s;
    value = select.attr("value");
    if (value){
        text = "";
        for (c in select.children()){
          if (select.children()[c].value == value){
            text = select.children()[c].innerHTML;
            break;
          }
        }
        select.before("" + text + "<input type=\\"hidden\\" name=\\"" + select.attr("name") + "\\" value=\\"" + value + "\\" \>");
        select.remove();
    }
    """ % {'select':select}

register.simple_tag(js_make_select_readonly)

def edit_subtypes(product):
    output = '
'
    subtypes = product.get_subtypes()
    for app, subtype in active_product_types():
        is_config = "ConfigurableProduct" in subtypes
        app_label = app.split(".")[-1]
        if subtype in subtypes:
            edit_url = reverse('admin:%s_%s_change' %
                               (app_label, subtype.lower()),
                                args=(product.pk,))
            output += ('
' % edit_url +
                       _('Edit %(subtype)s') % {'subtype': subtype} +
                       '
')
            if is_config or subtype=="ProductVariation":
                 output += '
Variation Manager
' % (reverse("satchmo_admin_variation_manager", args = [product.id]))
        else:
            if not(is_config and subtype=="ProductVariation"):
                add_url = reverse('admin:%s_%s_add' %
                                  (app_label, subtype.lower()))
                output += ('
' % (add_url, product.id) +
                           _('Add %(subtype)s') % {'subtype': subtype} +
                           '
')
    output += '
'
    return output

register.simple_tag(edit_subtypes)

def list_variations(configurableproduct):
    opts = configurableproduct.get_all_options()
    output = "{% load admin_modify adminmedia %}"
    output += "
"
    for p_opt in opts:
        opt_strs = []
        [opt_strs.append(opt.name) for opt in p_opt]
        opt_str = ', '.join(opt_strs)

        product = configurableproduct.get_product_from_options(p_opt)
        if product:
            p_url = '/admin/product/product/%s/' % product.pk
            pv_url = '/admin/product/productvariation/%s/delete/' % product.pk
            output += """

            %s
            %s
            %s

            """ % (opt_str, p_url, product.slug, pv_url,
                _("Delete ProductVariation"))
        else:
            #opt_pks = [str(opt.pk) for opt in p_opt]
            #opt_pks = ','.join(opt_pks)
            # TODO [NFA]: Blocked by Django ticket #7738.
            opt_pks = ''
            add_url = '/admin/product/productvariation/add/' + \
                "?product=%s&parent=%s&options=%s" % (
                configurableproduct.product.pk, configurableproduct.product.pk,
                opt_pks)
            output += """

            %s

              Add Variation

            """ % (opt_str, add_url)
    output += "
"
    t = Template(output)
    return t.render(Context())

register.simple_tag(list_variations)

<|endoftext|>


<commit_before># import atexit
import logging

import psycopg2
from psycopg2.extras import DictCursor
from psycopg2 import OperationalError, ProgrammingError, connect

from . import config

class DataSourceException(Exception):
    pass

class DataSourceBase(object):
    """
    Base class for querying geo spatial datasets
    """
    db = None
    dsn = None
    dataset = None
    default_properties = ('id', 'display', 'type', 'uri')
    radius = 30
    meta = {}
    use_rd = True
    x = None
    y = None

    def __init__(self):
        self.logger = logging.getLogger(self.__class__.__name__)
        self.logger.debug('Creating DataSource: %s' % self.dataset)

        if not self.dsn:
            raise ValueError('dsn needs to be defined')

    def get_cursor(self):
        return self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor)

    def execute_queries(self):
        try:
            conn = connect(self.dsn)
        except OperationalError as err:
            self.logger.error('Error creating connection: %s' % err)
            raise DataSourceException('error connecting to datasource')
            return

        features = []
        with self.get_cursor() as cur:
            for dataset in self.meta['datasets']:
                for dataset_ident, table in self.meta['datasets'][dataset].items():
                    if self.meta['operator'] == 'contains':
                        rows = self.execute_polygon_query(cur, table)
                    else:
                        rows = self.execute_point_query(cur, table)

                    if not len(rows):
                        self.logger.debug(table, 'no results')
                        continue

                    for row in rows:
                        features.append({
                            'properties': dict([
                                (prop, row[prop])
                                for prop in self.default_properties if prop in row])
                        })
        # Closing the connection to the db
        conn.close()

        return features

    def execute_point_query(self, cur, table):
        if not self.use_rd:
            sql = """
SELECT *
FROM {}
WHERE ST_DWithin({}, ST_Transform(ST_GeomFromText(\'POINT(%s %s)\', 4326), 28992), %s)
            """.format(
                table, self.meta['geofield']
            )
            cur.execute(sql, (self.x, self.y, self.radius))
        else:
            sql = """
SELECT *
FROM {}
WHERE ST_DWithin({}, ST_GeomFromText(\'POINT(%s %s)\', 28992), %s)
            """.format(
                table, self.meta['geofield']
            )
            cur.execute(sql, (self.x, self.y, self.radius))

        return cur.fetchall()

    def execute_polygon_query(self, cur, table):
        if not self.use_rd:
            sql = """
SELECT *
FROM {}
WHERE {} && ST_Transform(ST_GeomFromText(\'POINT(%s %s)\', 4326), 28992)
AND
ST_Contains({}, ST_Transform(ST_GeomFromText(\'POINT(%s %s)\', 4326), 28992))
            """.format(
                table, self.meta['geofield'], self.meta['geofield']
            )
            cur.execute(sql, (self.y, self.x)*2)
        else:
            sql = """
SELECT *
FROM {}
WHERE {} && ST_GeomFromText(\'POINT(%s %s)\', 28992)
AND
ST_Contains({}, ST_GeomFromText(\'POINT(%s %s)\', 28992))
            """.format(
                table, self.meta['geofield'], self.meta['geofield']
            )
            cur.execute(sql, (self.x, self.y)*2)

        return cur.fetchall()

class AtlasDataSource(DataSourceBase):
    dsn = config.DSN_ATLAS
    meta = {
        'geofield': 'geometrie',
        'operator': 'contains',
        'datasets': {
            'bag': {
                'openbareruimte': 'public.geo_bag_openbareruimte_mat',
                'pand': 'public.geo_bag_pand_mat',
                'ligplaats': 'public.geo_bag_ligplaats_mat',
                'standplaats': 'public.geo_bag_standplaats_mat',
            },
            'gebieden': {
                'stadsdeel': 'public.geo_bag_stadsdeel_mat',
                'buurt': 'public.geo_bag_buurt_mat',
                'buurtcombinatie': 'public.geo_bag_buurtcombinatie_mat',
                'bouwblok': 'public.geo_bag_bouwblok_mat',
                'grootstedelijkgebied': 'public.geo_bag_grootstedelijkgebied_mat',
                'gebiedsgerichtwerken': 'public.geo_bag_gebiedsgerichtwerken_mat',
                'unesco': 'public.geo_bag_unesco_mat',
            },
            'lki': {
                'kadastraal_object': 'public.geo_lki_kadastraalobject_mat',
            },
            'wkpb': {
                'beperking': 'public.geo_wkpb_mat',
            },
        },
    }

    def query(self, x, y, rd=True):
        self.use_rd = rd
        self.x = x
        self.y = y

        try:
            return {
                'type': 'FeatureCollection',
                'features': self.execute_queries()
            }
        except (DataSourceException, ProgrammingError) as err:
            return {
                'type': 'Error',
                'message': 'Error executing query: %s' % err.message
            }

class NapMeetboutenDataSource(DataSourceBase):
    dsn = config.DSN_NAP
    meta = {
        'geofield': 'geometrie',
        'operator': 'within',
        'datasets': {
            'nap': {
                'peilmerk': 'public.geo_nap_peilmerk_mat',
            },
            'meetbouten': {
                'meetbout': 'public.geo_meetbouten_meetbout_mat',
            },
        },
    }

    def query(self, x, y, radius=None, rd=True):
        self.use_rd = rd
        self.x = x
        self.y = y

        if radius:
            self.radius = radius

        try:
            return {
                'type': 'FeatureCollection',
                'features': self.execute_queries()
            }
        except (DataSourceException, ProgrammingError) as err:
            return {
                'type': 'Error',
                'message': 'Error executing query: %s' % err.message
            }
<commit_msg>Correcting reference to the connection variable<commit_after># import atexit
import logging

import psycopg2
from psycopg2.extras import DictCursor
from psycopg2 import OperationalError, ProgrammingError, connect

from . import config

class DataSourceException(Exception):
    pass

class DataSourceBase(object):
    """
    Base class for querying geo spatial datasets
    """
    db = None
    dsn = None
    dataset = None
    default_properties = ('id', 'display', 'type', 'uri')
    radius = 30
    meta = {}
    use_rd = True
    x = None
    y = None

    def __init__(self):
        self.logger = logging.getLogger(self.__class__.__name__)
        self.logger.debug('Creating DataSource: %s' % self.dataset)

        if not self.dsn:
            raise ValueError('dsn needs to be defined')

    def get_cursor(self, conn):
        return conn.cursor(cursor_factory=psycopg2.extras.DictCursor)

    def execute_queries(self):
        try:
            conn = connect(self.dsn)
        except OperationalError as err:
            self.logger.error('Error creating connection: %s' % err)
            raise DataSourceException('error connecting to datasource')
            return

        features = []
        with self.get_cursor(conn) as cur:
            for dataset in self.meta['datasets']:
                for dataset_ident, table in self.meta['datasets'][dataset].items():
                    if self.meta['operator'] == 'contains':
                        rows = self.execute_polygon_query(cur, table)
                    else:
                        rows = self.execute_point_query(cur, table)

                    if not len(rows):
                        self.logger.debug(table, 'no results')
                        continue

                    for row in rows:
                        features.append({
                            'properties': dict([
                                (prop, row[prop])
                                for prop in self.default_properties if prop in row])
                        })
        # Closing the connection to the db
        conn.close()

        return features

    def execute_point_query(self, cur, table):
        if not self.use_rd:
            sql = """
SELECT *
FROM {}
WHERE ST_DWithin({}, ST_Transform(ST_GeomFromText(\'POINT(%s %s)\', 4326), 28992), %s)
            """.format(
                table, self.meta['geofield']
            )
            cur.execute(sql, (self.x, self.y, self.radius))
        else:
            sql = """
SELECT *
FROM {}
WHERE ST_DWithin({}, ST_GeomFromText(\'POINT(%s %s)\', 28992), %s)
            """.format(
                table, self.meta['geofield']
            )
            cur.execute(sql, (self.x, self.y, self.radius))

        return cur.fetchall()

    def execute_polygon_query(self, cur, table):
        if not self.use_rd:
            sql = """
SELECT *
FROM {}
WHERE {} && ST_Transform(ST_GeomFromText(\'POINT(%s %s)\', 4326), 28992)
AND
ST_Contains({}, ST_Transform(ST_GeomFromText(\'POINT(%s %s)\', 4326), 28992))
            """.format(
                table, self.meta['geofield'], self.meta['geofield']
            )
            cur.execute(sql, (self.y, self.x)*2)
        else:
            sql = """
SELECT *
FROM {}
WHERE {} && ST_GeomFromText(\'POINT(%s %s)\', 28992)
AND
ST_Contains({}, ST_GeomFromText(\'POINT(%s %s)\', 28992))
            """.format(
                table, self.meta['geofield'], self.meta['geofield']
            )
            cur.execute(sql, (self.x, self.y)*2)

        return cur.fetchall()

class AtlasDataSource(DataSourceBase):
    dsn = config.DSN_ATLAS
    meta = {
        'geofield': 'geometrie',
        'operator': 'contains',
        'datasets': {
            'bag': {
                'openbareruimte': 'public.geo_bag_openbareruimte_mat',
                'pand': 'public.geo_bag_pand_mat',
                'ligplaats': 'public.geo_bag_ligplaats_mat',
                'standplaats': 'public.geo_bag_standplaats_mat',
            },
            'gebieden': {
                'stadsdeel': 'public.geo_bag_stadsdeel_mat',
                'buurt': 'public.geo_bag_buurt_mat',
                'buurtcombinatie': 'public.geo_bag_buurtcombinatie_mat',
                'bouwblok': 'public.geo_bag_bouwblok_mat',
                'grootstedelijkgebied': 'public.geo_bag_grootstedelijkgebied_mat',
                'gebiedsgerichtwerken': 'public.geo_bag_gebiedsgerichtwerken_mat',
                'unesco': 'public.geo_bag_unesco_mat',
            },
            'lki': {
                'kadastraal_object': 'public.geo_lki_kadastraalobject_mat',
            },
            'wkpb': {
                'beperking': 'public.geo_wkpb_mat',
            },
        },
    }

    def query(self, x, y, rd=True):
        self.use_rd = rd
        self.x = x
        self.y = y

        try:
            return {
                'type': 'FeatureCollection',
                'features': self.execute_queries()
            }
        except (DataSourceException, ProgrammingError) as err:
            return {
                'type': 'Error',
                'message': 'Error executing query: %s' % err.message
            }

class NapMeetboutenDataSource(DataSourceBase):
    dsn = config.DSN_NAP
    meta = {
        'geofield': 'geometrie',
        'operator': 'within',
        'datasets': {
            'nap': {
                'peilmerk': 'public.geo_nap_peilmerk_mat',
            },
            'meetbouten': {
                'meetbout': 'public.geo_meetbouten_meetbout_mat',
            },
        },
    }

    def query(self, x, y, radius=None, rd=True):
        self.use_rd = rd
        self.x = x
        self.y = y

        if radius:
            self.radius = radius

        try:
            return {
                'type': 'FeatureCollection',
                'features': self.execute_queries()
            }
        except (DataSourceException, ProgrammingError) as err:
            return {
                'type': 'Error',
                'message': 'Error executing query: %s' % err.message
            }
<|endoftext|>


<commit_before>#    This file is part of rhizi, a collaborative knowledge graph editor.
#    Copyright (C) 2014-2015  Rhizi
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import base64
import gzip
import json

class RZCommit():
    """
    Rhizi Commit object

    Currently this object is substituted by either a diff object, this currently
    acts as a model stub.
    """

    @staticmethod
    def diff_obj_from_blob(blob):
        """
        @return json.loads(gzip_decompress(base64_decode(blob)))
        """
        blob_gzip = base64.decodestring(blob)
        blob = gzip.zlib.decompress(blob_gzip)
        return json.loads(blob)

    @staticmethod
    def blob_from_diff_obj(obj):
        """
        @return: blob = base64(gzip(json.dumps(obj)))
        """
        obj_str = json.dumps(obj)
        blob_gzip = gzip.zlib.compress(obj_str)
        blob_base64 = base64.encodestring(blob_gzip)

        return blob_base64

class Link():
    """
    documentation anchor - this class currently carries no implementation
    and only acts as a documentation anchor

    link['__src'] - meta attribute for link source object
    link['__dst'] - meta attribute for link destination object
    """

    def __init__(self, src=None, dst=None):
        assert False, 'currently unused'

    class Link_Ptr(dict):
        """
        link['__src_id'] - meta attribute for link source id
        link['__dst_id'] - meta attribute for link destination id
        """
        def __init__(self, src_id=None, dst_id=None):
            assert None is not src_id and None is not dst_id
            self['__src_id'] = src_id
            self['__dst_id'] = dst_id

        @property
        def src_id(self):
            return self['__src_id']

        @property
        def dst_id(self):
            return self['__dst_id']

    @staticmethod
    def link_ptr(src_id=None, dst_id=None):
        """
        init from src_id or dst_id attributes - at least one must be provided
        """
        return Link.Link_Ptr(src_id, dst_id)

class RZDoc():

    def __init__(self, rzdoc_name=None):
        self.id = None  # set upon DB commit
        self.name = rzdoc_name

    def __eq__(self, other):
        if not isinstance(other, RZDoc): return False

        assert self.id != None and other.id != None

        return self.id == other.id

    def __hash__(self):
        assert self.id != None

        return self.id.__hash__()

    def __str__(self):
        return 'rzdoc: name: %s' % (self.name)
<commit_msg>rhizi: python3: base64 & json changes<commit_after>#    This file is part of rhizi, a collaborative knowledge graph editor.
#    Copyright (C) 2014-2015  Rhizi
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import base64
import gzip
import json

class RZCommit():
    """
    Rhizi Commit object

    Currently this object is substituted by either a diff object, this currently
    acts as a model stub.
    """

    @staticmethod
    def diff_obj_from_blob(blob):
        """
        @return json.loads(gzip_decompress(base64_decode(blob)))
        """
        blob_gzip = base64.decodestring(blob.encode('utf-8'))
        blob = gzip.zlib.decompress(blob_gzip).decode()
        return json.loads(blob)

    @staticmethod
    def blob_from_diff_obj(obj):
        """
        @return: blob = base64(gzip(json.dumps(obj)))
        """
        obj_str = json.dumps(obj)
        blob_gzip = gzip.zlib.compress(obj_str)
        blob_base64 = base64.encodestring(blob_gzip)

        return blob_base64

class Link():
    """
    documentation anchor - this class currently carries no implementation
    and only acts as a documentation anchor

    link['__src'] - meta attribute for link source object
    link['__dst'] - meta attribute for link destination object
    """

    def __init__(self, src=None, dst=None):
        assert False, 'currently unused'

    class Link_Ptr(dict):
        """
        link['__src_id'] - meta attribute for link source id
        link['__dst_id'] - meta attribute for link destination id
        """
        def __init__(self, src_id=None, dst_id=None):
            assert None is not src_id and None is not dst_id
            self['__src_id'] = src_id
            self['__dst_id'] = dst_id

        @property
        def src_id(self):
            return self['__src_id']

        @property
        def dst_id(self):
            return self['__dst_id']

    @staticmethod
    def link_ptr(src_id=None, dst_id=None):
        """
        init from src_id or dst_id attributes - at least one must be provided
        """
        return Link.Link_Ptr(src_id, dst_id)

class RZDoc():

    def __init__(self, rzdoc_name=None):
        self.id = None  # set upon DB commit
        self.name = rzdoc_name

    def __eq__(self, other):
        if not isinstance(other, RZDoc): return False

        assert self.id != None and other.id != None

        return self.id == other.id

    def __hash__(self):
        assert self.id != None

        return self.id.__hash__()

    def __str__(self):
        return 'rzdoc: name: %s' % (self.name)
<|endoftext|>


<commit_before>## \file
## \ingroup tutorial_pyroot
## Display two histograms and their ratio.
##
## This program illustrates how to plot two histograms and their
## ratio on the same canvas. Original macro by Olivier Couet.
##
## \macro_code
##
## \author Michael Moran

from ROOT import TCanvas, TColor, TGaxis, TH1F, TPad
from ROOT import kBlack, kBlue, kRed

def createH1():
  h1 = TH1F("h1", ("Two gaussian plots and their ratio; x title; h1 and h2"
            " histograms"), 100, -5, 5)
  h1.SetLineColor(kBlue+1)
  h1.SetLineWidth(2)
  h1.FillRandom("gaus")
  h1.GetYaxis().SetTitleSize(20)
  h1.GetYaxis().SetTitleFont(43)
  h1.GetYaxis().SetTitleOffset(1.55)
  h1.SetStats(0)
  return h1

def createH2():
  h2 = TH1F("h2", "h2", 100, -5, 5)
  h2.FillRandom("gaus")
  h2.SetLineColor(kRed)
  h2.SetLineWidth(2)
  return h2

def createRatio(h1, h2):
  h3 = h1.Clone("h3")
  h3.SetLineColor(kBlack)
  h3.SetMarkerStyle(21)
  h3.SetTitle("")
  h3.SetMinimum(0.8)
  h3.SetMaximum(1.35)
  # Set up plot for markers and errors
  h3.Sumw2()
  h3.SetStats(0)
  h3.Divide(h2)

  # Adjust y-axis settings
  y = h3.GetYaxis()
  y.SetTitle("ratio h1/h2 ")
  y.SetNdivisions(505)
  y.SetTitleSize(20)
  y.SetTitleFont(43)
  y.SetTitleOffset(1.55)
  y.SetLabelFont(43)
  y.SetLabelSize(15)

  # Adjust x-axis settings
  x = h3.GetXaxis()
  x.SetTitleSize(20)
  x.SetTitleFont(43)
  x.SetTitleOffset(4.0)
  x.SetLabelFont(43)
  x.SetLabelSize(15)

  return h3

def createCanvasPads():
  c = TCanvas("c", "canvas", 800, 800)
  # Upper histogram plot is pad1
  pad1 = TPad("pad1", "pad1", 0, 0.3, 1, 1.0)
  pad1.SetBottomMargin(0)  # joins upper and lower plot
  pad1.SetGridx()
  pad1.Draw()
  # Lower ratio plot is pad2
  c.cd()  # returns to main canvas before defining pad2
  pad2 = TPad("pad2", "pad2", 0, 0.05, 1, 0.3)
  pad2.SetTopMargin(0)  # joins upper and lower plot
  pad2.SetBottomMargin(0.2)
  pad2.SetGridx()
  pad2.Draw()

  return c, pad1, pad2

def ratioplot():
  # create required parts
  h1 = createH1()
  h2 = createH2()
  h3 = createRatio(h1, h2)
  c, pad1, pad2 = createCanvasPads()

  # draw everything
  pad1.cd()
  h1.Draw()
  h2.Draw("same")
  # to avoid clipping the bottom zero, redraw a small axis
  h1.GetYaxis().SetLabelSize(0.0)
  axis = TGaxis(-5, 20, -5, 220, 20, 220, 510, "")
  axis.SetLabelFont(43)
  axis.SetLabelSize(15)
  axis.Draw()
  pad2.cd()
  h3.Draw("ep")

  # To hold window open when running from command line
#  text = raw_input()

if __name__ == "__main__":
  ratioplot()
<commit_msg>Change permissions<commit_after>## \file
## \ingroup tutorial_pyroot
## Display two histograms and their ratio.
##
## This program illustrates how to plot two histograms and their
## ratio on the same canvas. Original macro by Olivier Couet.
##
## \macro_code
##
## \author Michael Moran

from ROOT import TCanvas, TColor, TGaxis, TH1F, TPad
from ROOT import kBlack, kBlue, kRed

def createH1():
  h1 = TH1F("h1", ("Two gaussian plots and their ratio; x title; h1 and h2"
            " histograms"), 100, -5, 5)
  h1.SetLineColor(kBlue+1)
  h1.SetLineWidth(2)
  h1.FillRandom("gaus")
  h1.GetYaxis().SetTitleSize(20)
  h1.GetYaxis().SetTitleFont(43)
  h1.GetYaxis().SetTitleOffset(1.55)
  h1.SetStats(0)
  return h1

def createH2():
  h2 = TH1F("h2", "h2", 100, -5, 5)
  h2.FillRandom("gaus")
  h2.SetLineColor(kRed)
  h2.SetLineWidth(2)
  return h2

def createRatio(h1, h2):
  h3 = h1.Clone("h3")
  h3.SetLineColor(kBlack)
  h3.SetMarkerStyle(21)
  h3.SetTitle("")
  h3.SetMinimum(0.8)
  h3.SetMaximum(1.35)
  # Set up plot for markers and errors
  h3.Sumw2()
  h3.SetStats(0)
  h3.Divide(h2)

  # Adjust y-axis settings
  y = h3.GetYaxis()
  y.SetTitle("ratio h1/h2 ")
  y.SetNdivisions(505)
  y.SetTitleSize(20)
  y.SetTitleFont(43)
  y.SetTitleOffset(1.55)
  y.SetLabelFont(43)
  y.SetLabelSize(15)

  # Adjust x-axis settings
  x = h3.GetXaxis()
  x.SetTitleSize(20)
  x.SetTitleFont(43)
  x.SetTitleOffset(4.0)
  x.SetLabelFont(43)
  x.SetLabelSize(15)

  return h3

def createCanvasPads():
  c = TCanvas("c", "canvas", 800, 800)
  # Upper histogram plot is pad1
  pad1 = TPad("pad1", "pad1", 0, 0.3, 1, 1.0)
  pad1.SetBottomMargin(0)  # joins upper and lower plot
  pad1.SetGridx()
  pad1.Draw()
  # Lower ratio plot is pad2
  c.cd()  # returns to main canvas before defining pad2
  pad2 = TPad("pad2", "pad2", 0, 0.05, 1, 0.3)
  pad2.SetTopMargin(0)  # joins upper and lower plot
  pad2.SetBottomMargin(0.2)
  pad2.SetGridx()
  pad2.Draw()

  return c, pad1, pad2

def ratioplot():
  # create required parts
  h1 = createH1()
  h2 = createH2()
  h3 = createRatio(h1, h2)
  c, pad1, pad2 = createCanvasPads()

  # draw everything
  pad1.cd()
  h1.Draw()
  h2.Draw("same")
  # to avoid clipping the bottom zero, redraw a small axis
  h1.GetYaxis().SetLabelSize(0.0)
  axis = TGaxis(-5, 20, -5, 220, 20, 220, 510, "")
  axis.SetLabelFont(43)
  axis.SetLabelSize(15)
  axis.Draw()
  pad2.cd()
  h3.Draw("ep")

  # To hold window open when running from command line
#  text = raw_input()

if __name__ == "__main__":
  ratioplot()
<|endoftext|>


<commit_before>from contextlib import contextmanager
from datetime import datetime

from tveebot_tracker.episode import TVShow, Quality, Episode, State

class Connection:
    """ Abstraction for a connection for the Episode DB """

    def commit(self):
        pass

    def rollback(self):
        pass

    def close(self):
        pass

    #
    # TV Show Table methods
    #

    def insert_tvshow(self, tvshow: TVShow, quality: Quality):
        pass

    def delete_tvshow(self, tvshow_id: str):
        pass

    def tvshows(self):
        pass

    def set_tvshow_quality(self, tvshow_id: str, quality: Quality):
        pass

    #
    # Episode Table Methods
    #

    def insert_episode(self, episode: Episode):
        pass

    def episodes(self):
        pass

    def episodes_from(self, tvshow: TVShow):
        pass

    def set_episode_state(self, episode: Episode, state: State):
        pass

    def set_episode_quality(self, episode: Episode, quality: Quality):
        pass

    def set_episode_download_timestamp(self, episode: Episode, download_timestamp: datetime):
        pass

class EpisodeDB:
    """ Abstraction for the Episode DB """

    @contextmanager
    def connect(self):
        pass
<commit_msg>Connection is can now be used in 'with' statements<commit_after>from datetime import datetime

from tveebot_tracker.episode import TVShow, Quality, Episode, State

class EpisodeDB:
    """ Abstraction for the Episode DB """

class Connection:
    """ Abstraction for a connection for the Episode DB """

    def __init__(self, database: EpisodeDB):
        self._db = database

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()

    def commit(self):
        pass

    def rollback(self):
        pass

    def close(self):
        pass

    #
    # TV Show Table methods
    #

    def insert_tvshow(self, tvshow: TVShow, quality: Quality):
        pass

    def delete_tvshow(self, tvshow_id: str):
        pass

    def tvshows(self):
        pass

    def set_tvshow_quality(self, tvshow_id: str, quality: Quality):
        pass

    #
    # Episode Table Methods
    #

    def insert_episode(self, episode: Episode):
        pass

    def episodes(self):
        pass

    def episodes_from(self, tvshow: TVShow):
        pass

    def set_episode_state(self, episode: Episode, state: State):
        pass

    def set_episode_quality(self, episode: Episode, quality: Quality):
        pass

    def set_episode_download_timestamp(self, episode: Episode,
                                       download_timestamp: datetime):
        pass

def connect(db: EpisodeDB) -> Connection:
    """ Returns a connection to the DB """
    return Connection(db)
<|endoftext|>


<commit_before>from PIL import Image
import math
from random import randint

def gif(sauce, frameCount, targetRect):
    """Given a source image `sauce`, return `frameCount` frames
    into `targetRect` target window.
    Returns iterable of PIL.Image"""

    fullSize = sauce.size
    targetSize = (targetRect[2]-targetRect[0], targetRect[3]-targetRect[1])
    ratio = (1.0*fullSize[0]/targetSize[0], 1.0*fullSize[1]/targetSize[1])
    targetCenter = (targetRect[0]/(1 - 1/ratio[0]), targetRect[1]/(1 - 1/ratio[1]))

    def scale(i):
        factor = (math.pow(ratio[0], 1.0*i/frameCount),
                    math.pow(ratio[1], 1.0*i/frameCount))
        return tuple(map(lambda x: x,
                (targetCenter[0] - targetCenter[0]/ratio[0]*factor[0],
                 targetCenter[1] - targetCenter[1]/ratio[1]*factor[1],
                 targetSize[0] * factor[0],
                 targetSize[1] * factor[1])))

    for frame in range(frameCount):
        dest = Image.new(sauce.mode, fullSize, 255*frame/frameCount)
        iterations = 1000 # maximum number of iterations allowed per frame
        i = frame + frameCount
        rectum = scale(i)
        while rectum[2] > 1 and rectum[3] > 1 and iterations > 0:
            draw(sauce, rectum, dest)
            i -= frameCount
            rectum = scale(i)
            iterations -= 1

        yield dest
    print targetRect, sauce.size, ratio, targetCenter

def clamp(rectum, dest):
    left, top, w, h = int(rectum[0]), int(rectum[1]), int(rectum[2]), int(rectum[3])
    return (w,h), (left, top, left+w, top+h)

def draw(sauce, target, dest):
    size, box = clamp(target, dest)
    matrix = (sauce.size[0]/target[2], 0, 0,
              0, sauce.size[1]/target[3], 0)
    scaled = sauce.transform(size, Image.AFFINE, matrix)
    dest.paste(scaled, box)
<commit_msg>don't bother outputting random color underneath<commit_after>from PIL import Image
import math
from random import randint

def gif(sauce, frameCount, targetRect):
    """Given a source image `sauce`, return `frameCount` frames
    into `targetRect` target window.
    Returns iterable of PIL.Image"""

    fullSize = sauce.size
    targetSize = (targetRect[2]-targetRect[0], targetRect[3]-targetRect[1])
    ratio = (1.0*fullSize[0]/targetSize[0], 1.0*fullSize[1]/targetSize[1])
    targetCenter = (targetRect[0]/(1 - 1/ratio[0]), targetRect[1]/(1 - 1/ratio[1]))

    def scale(i):
        factor = (math.pow(ratio[0], 1.0*i/frameCount),
                    math.pow(ratio[1], 1.0*i/frameCount))
        return tuple(map(lambda x: x,
                (targetCenter[0] - targetCenter[0]/ratio[0]*factor[0],
                 targetCenter[1] - targetCenter[1]/ratio[1]*factor[1],
                 targetSize[0] * factor[0],
                 targetSize[1] * factor[1])))

    for frame in range(frameCount):
        dest = Image.new(sauce.mode, fullSize)
        iterations = 1000 # maximum number of iterations allowed per frame
        i = frame + frameCount
        rectum = scale(i)
        while rectum[2] > 1 and rectum[3] > 1 and iterations > 0:
            draw(sauce, rectum, dest)
            i -= frameCount
            rectum = scale(i)
            iterations -= 1

        yield dest
    print targetRect, sauce.size, ratio, targetCenter

def clamp(rectum, dest):
    left, top, w, h = int(rectum[0]), int(rectum[1]), int(rectum[2]), int(rectum[3])
    return (w,h), (left, top, left+w, top+h)

def draw(sauce, target, dest):
    size, box = clamp(target, dest)
    matrix = (sauce.size[0]/target[2], 0, 0,
              0, sauce.size[1]/target[3], 0)
    scaled = sauce.transform(size, Image.AFFINE, matrix)
    dest.paste(scaled, box)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2012 Splunk, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"): you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, sofaare
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""
Scripted input that streams data from Twitter's 1% sample feed to standard
output, with each event on a separate line in JSON format.

The original JSON from Twitter is augmented with an extra '__time' key on each
event. This makes it possible for Splunk to use a regex to parse out the
timestamp accurately.

This script can be used as a standalone command-line program or imported as a
module and invoked directly via either the `start` method or the
`read_sample_tweet_stream` method.
"""

from twython import TwythonStreamer

import json
import splunk
import splunklib.client
import sys
import traceback

class TweetStreamer(TwythonStreamer):
    """
    Collects and writes a stream of tweets from Twitter
    """

    def __init__(self, app_key, app_secret, oauth_token, oauth_token_secret,
                 timeout=300, retry_count=None, retry_in=10, headers=None,
                 output_stream=None):
        super(TweetStreamer, self).__init__(
            app_key, app_secret, oauth_token, oauth_token_secret, timeout,
            retry_count, retry_in, headers)
        self._output_stream = \
            sys.stdout if output_stream is None else output_stream

    def on_error(self, status_code, data):
        sys.stderr.write(
            'Twitter reported status code %d: %s' % (status_code, data))
        sys.stderr.flush()

    def on_limit(self, limit):
        sys.stderr.write(
            'More Tweets matched than the Twitter rate limit allows. %d ' +
            'undelivered Tweets since the connection was opened.\n' %
            limit['track'])

    def on_success(self, data):
        # Super calls our other handlers using rules we'd like not to duplicate
        super(TweetStreamer, self).on_success(data)

        if 'created_at' in data:
            # Generate a synthetic field with the desired timestamp so that
            # Splunk can accurately parse it
            data['__time'] = data['created_at']

        # Preserve compact output format with separators
        json.dump(data, self._output_stream, separators=(',', ':'))

        self._output_stream.write('\r\n')

def get_oauth_settings(session_key):
    """
    Retrieves OAuth settings from args.

    :param session_key: A value identifying the current Splunk session.
    :return: OAuth settings tuple: 'app_key', 'app_secret', 'oauth_token', and
    'oauth_token_secret'
    """

    # NOTE: Requires 'develop' version of splunklib past 0.8.0 for 'token'
    # parameter to be honored
    service = splunklib.client.Service(
        host=splunk.getDefault('host'),
        port=splunk.getDefault('port'),
        scheme=splunk.getDefault('protocol'),
        owner='nobody',
        app='twitter2',
        token='Splunk %s' % session_key)

    passwords = splunklib.client.Collection(service, 'admin/passwords')
    oauth_settings = None

    for password in passwords:
        oauth_settings = password
        break

    if oauth_settings is None:
        sys.stderr.write(
            'Could not retrieve Twitter OAuth settings from Splunk. Set up ' +
            'the app to correct this issue.\n')
        exit(1)

    app_key = oauth_settings.content['realm']
    oauth_token = oauth_settings.content['username']
    secrets = oauth_settings.content['clear_password']

    if None in (app_key, oauth_token, secrets):
        sys.stderr.write(
            'Splunk returned incomplete Twitter OAuth settings. Set up the ' +
            'app to correct this issue.\n')
        exit(1)

    try:
        (app_secret, oauth_token_secret) = secrets.split(':')
    except ValueError as e:
        sys.stderr.write(
            'Could not parse the Twitter OAuth secrets returned by Splunk: '
            '%s. Set up the app to correct this issue.\n' % e.message)
        exit(1)

    return app_key, app_secret, oauth_token, oauth_token_secret

def read_sample_tweet_stream(
    app_key, app_secret, oauth_token, oauth_token_secret, output_stream=None
):
    """
    Continuously reads tweets from the sample Twitter stream and writes them to
    the specified output stream or `sys.stdout`, if no stream is provided.

    Never returns, unless an I/O error occurs.

    :param app_key:                 Twitter application consumer key
    :param app_secret:              Twitter application consumer secret
    :param oauth_token:             Twitter application access token
    :param oauth_token_secret:      Twitter application access token secret
    :param output_stream:           (optional) A file-like object to which data
                                    will be written.
                                    Must support a `write` method. Defaults to
                                    `sys.stdout`.
    :raises KeyboardInterrupt       If you stop the tweet stream using
    Control-C.
    :raises TwythonAuthError:       If the tweet stream stops due to some issue
                                    with your authentication.
    :raises TwythonRateLimitError:  If the tweet stream stops because you hit a
                                    rate limit.
    :raises TwythonError:           If the tweet stream stops for any other
                                    reason.
    """

    streamer = TweetStreamer(
        app_key=app_key,
        app_secret=app_secret,
        oauth_token=oauth_token,
        oauth_token_secret=oauth_token_secret,
        output_stream=output_stream
    )

    streamer.statuses.sample()

def start(app_key, app_secret, oauth_token, oauth_token_secret,
          output_stream=None):
    """
    Continuously reads tweets from the sample Twitter stream and writes them to
    the specified output stream or `sys.stdout`, if no stream is provided.

    Never returns, unless an I/O error occurs.

    :param app_key:                 Twitter application consumer key
    :param app_secret:              Twitter application consumer secret
    :param oauth_token:             Twitter application access token
    :param oauth_token_secret:      Twitter application access token secret
    :param output_stream:           (optional) A file-like object to which data
                                    will be written.
                                    Must support a `write` method. Defaults to
                                    `sys.stdout`.
    :raises TwythonAuthError:       If the tweet stream stops due to some issue
                                    with your authentication.
    :raises TwythonRateLimitError:  If the tweet stream stops because you hit a
                                    rate limit.
    :raises TwythonError:           If the tweet stream stops for any other
                                    reason.
    """

    try:
        read_sample_tweet_stream(
            app_key, app_secret, oauth_token, oauth_token_secret, output_stream)
    except KeyboardInterrupt:
        pass
    except Exception:
        traceback.print_exc(file=sys.stderr)
        sys.stderr.flush()
        raise

def main():
    session_key = sys.stdin.readline().strip()

    if len(session_key) == 0:
        sys.stderr.write(
            'Did not receive a session key from splunkd. Please enable ' +
            'passAuth in inputs.conf for this script.\n')
        exit(2)

    app_key, app_secret, oauth_token, oauth_token_secret = \
        get_oauth_settings(session_key)

    start(
        app_key=app_key,
        app_secret=app_secret,
        oauth_token=oauth_token,
        oauth_token_secret=oauth_token_secret)

if __name__ == '__main__':
    main()
<commit_msg>Workaround DVPL-2401: Index Unicode using unescaped JSON.<commit_after>#!/usr/bin/env python
#
# Copyright 2012 Splunk, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"): you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, sofaare
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""
Scripted input that streams data from Twitter's 1% sample feed to standard
output, with each event on a separate line in JSON format.

The original JSON from Twitter is augmented with an extra '__time' key on each
event. This makes it possible for Splunk to use a regex to parse out the
timestamp accurately.

This script can be used as a standalone command-line program or imported as a
module and invoked directly via either the `start` method or the
`read_sample_tweet_stream` method.
"""

from twython import TwythonStreamer

import json
import splunk
import splunklib.client
import sys
import traceback

class TweetStreamer(TwythonStreamer):
    """
    Collects and writes a stream of tweets from Twitter
    """

    def __init__(self, app_key, app_secret, oauth_token, oauth_token_secret,
                 timeout=300, retry_count=None, retry_in=10, headers=None,
                 output_stream=None):
        super(TweetStreamer, self).__init__(
            app_key, app_secret, oauth_token, oauth_token_secret, timeout,
            retry_count, retry_in, headers)
        self._output_stream = \
            sys.stdout if output_stream is None else output_stream

    def on_error(self, status_code, data):
        sys.stderr.write(
            'Twitter reported status code %d: %s' % (status_code, data))
        sys.stderr.flush()

    def on_limit(self, limit):
        sys.stderr.write(
            'More Tweets matched than the Twitter rate limit allows. %d ' +
            'undelivered Tweets since the connection was opened.\n' %
            limit['track'])

    def on_success(self, data):
        # Super calls our other handlers using rules we'd like not to duplicate
        super(TweetStreamer, self).on_success(data)

        if 'created_at' in data:
            # Generate a synthetic field with the desired timestamp so that
            # Splunk can accurately parse it
            data['__time'] = data['created_at']

        # Preserve compact output format with separators
        json.dump(data, self._output_stream,
            # Preserve compact output format
            separators=(',', ':'),
            # Workaround issue where searching for \u####-encoded characters
            # doesn't work. (DVPL-2401)
            ensure_ascii=False)

        self._output_stream.write('\r\n')

def get_oauth_settings(session_key):
    """
    Retrieves OAuth settings from args.

    :param session_key: A value identifying the current Splunk session.
    :return: OAuth settings tuple: 'app_key', 'app_secret', 'oauth_token', and
    'oauth_token_secret'
    """

    # NOTE: Requires 'develop' version of splunklib past 0.8.0 for 'token'
    # parameter to be honored
    service = splunklib.client.Service(
        host=splunk.getDefault('host'),
        port=splunk.getDefault('port'),
        scheme=splunk.getDefault('protocol'),
        owner='nobody',
        app='twitter2',
        token='Splunk %s' % session_key)

    passwords = splunklib.client.Collection(service, 'admin/passwords')
    oauth_settings = None

    for password in passwords:
        oauth_settings = password
        break

    if oauth_settings is None:
        sys.stderr.write(
            'Could not retrieve Twitter OAuth settings from Splunk. Set up ' +
            'the app to correct this issue.\n')
        exit(1)

    app_key = oauth_settings.content['realm']
    oauth_token = oauth_settings.content['username']
    secrets = oauth_settings.content['clear_password']

    if None in (app_key, oauth_token, secrets):
        sys.stderr.write(
            'Splunk returned incomplete Twitter OAuth settings. Set up the ' +
            'app to correct this issue.\n')
        exit(1)

    try:
        (app_secret, oauth_token_secret) = secrets.split(':')
    except ValueError as e:
        sys.stderr.write(
            'Could not parse the Twitter OAuth secrets returned by Splunk: '
            '%s. Set up the app to correct this issue.\n' % e.message)
        exit(1)

    return app_key, app_secret, oauth_token, oauth_token_secret

def read_sample_tweet_stream(
    app_key, app_secret, oauth_token, oauth_token_secret, output_stream=None
):
    """
    Continuously reads tweets from the sample Twitter stream and writes them to
    the specified output stream or `sys.stdout`, if no stream is provided.

    Never returns, unless an I/O error occurs.

    :param app_key:                 Twitter application consumer key
    :param app_secret:              Twitter application consumer secret
    :param oauth_token:             Twitter application access token
    :param oauth_token_secret:      Twitter application access token secret
    :param output_stream:           (optional) A file-like object to which data
                                    will be written.
                                    Must support a `write` method. Defaults to
                                    `sys.stdout`.
    :raises KeyboardInterrupt       If you stop the tweet stream using
    Control-C.
    :raises TwythonAuthError:       If the tweet stream stops due to some issue
                                    with your authentication.
    :raises TwythonRateLimitError:  If the tweet stream stops because you hit a
                                    rate limit.
    :raises TwythonError:           If the tweet stream stops for any other
                                    reason.
    """

    streamer = TweetStreamer(
        app_key=app_key,
        app_secret=app_secret,
        oauth_token=oauth_token,
        oauth_token_secret=oauth_token_secret,
        output_stream=output_stream
    )

    streamer.statuses.sample()

def start(app_key, app_secret, oauth_token, oauth_token_secret,
          output_stream=None):
    """
    Continuously reads tweets from the sample Twitter stream and writes them to
    the specified output stream or `sys.stdout`, if no stream is provided.

    Never returns, unless an I/O error occurs.

    :param app_key:                 Twitter application consumer key
    :param app_secret:              Twitter application consumer secret
    :param oauth_token:             Twitter application access token
    :param oauth_token_secret:      Twitter application access token secret
    :param output_stream:           (optional) A file-like object to which data
                                    will be written.
                                    Must support a `write` method. Defaults to
                                    `sys.stdout`.
    :raises TwythonAuthError:       If the tweet stream stops due to some issue
                                    with your authentication.
    :raises TwythonRateLimitError:  If the tweet stream stops because you hit a
                                    rate limit.
    :raises TwythonError:           If the tweet stream stops for any other
                                    reason.
    """

    try:
        read_sample_tweet_stream(
            app_key, app_secret, oauth_token, oauth_token_secret, output_stream)
    except KeyboardInterrupt:
        pass
    except Exception:
        traceback.print_exc(file=sys.stderr)
        sys.stderr.flush()
        raise

def main():
    session_key = sys.stdin.readline().strip()

    if len(session_key) == 0:
        sys.stderr.write(
            'Did not receive a session key from splunkd. Please enable ' +
            'passAuth in inputs.conf for this script.\n')
        exit(2)

    app_key, app_secret, oauth_token, oauth_token_secret = \
        get_oauth_settings(session_key)

    start(
        app_key=app_key,
        app_secret=app_secret,
        oauth_token=oauth_token,
        oauth_token_secret=oauth_token_secret)

if __name__ == '__main__':
    main()
<|endoftext|>