<commit_before>"""
Copyright (c) 2016 GP Garcia, CoaguCo Industries

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
__version__ = '2.0.0'
__author__  = 'GP Garcia'

import sys, os, time
from ctypes import *
from enum import Enum

import steamworks.util 		as s_util
from steamworks.enums 		import *
from steamworks.structs 	import *
from steamworks.exceptions 	import *
from steamworks.methods 	import STEAMWORKS_METHODS

from steamworks.interfaces.apps         import SteamApps
from steamworks.interfaces.friends      import SteamFriends
from steamworks.interfaces.matchmaking  import SteamMatchmaking
from steamworks.interfaces.music        import SteamMusic
from steamworks.interfaces.screenshots  import SteamScreenshots
from steamworks.interfaces.users        import SteamUsers
from steamworks.interfaces.userstats    import SteamUserStats
from steamworks.interfaces.utils        import SteamUtils
from steamworks.interfaces.workshop     import SteamWorkshop

os.environ['LD_LIBRARY_PATH'] = os.getcwd()

class STEAMWORKS(object):
    """
        Primary STEAMWORKS class used for fundamental handling of the STEAMWORKS API
    """
    _arch = s_util.get_arch()
    _native_supported_platforms = ['linux', 'linux2', 'darwin', 'win32']

    def __init__(self, supported_platforms: list = []) -> None:
        self._supported_platforms = supported_platforms
        self._loaded 	= False
        self._cdll 		= None

        self.app_id 	= 0

        self._initialize()

    def _initialize(self) -> bool:
        """Initialize module by loading STEAMWORKS library

        :return: bool
        """
        platform = sys.platform
        if self._supported_platforms and platform not in self._supported_platforms:
            raise UnsupportedPlatformException(f'"{platform}" has been excluded')

        if platform not in STEAMWORKS._native_supported_platforms:
            raise UnsupportedPlatformException(f'"{platform}" is not being supported')

        library_file_name = ''
        if platform in ['linux', 'linux2']:
            library_file_name = 'SteamworksPy.so'

        elif platform == 'darwin':
            library_file_name = 'SteamworksPy.dylib'

        elif platform == 'win32':
            library_file_name = 'SteamworksPy.dll' if STEAMWORKS._arch == Arch.x86 else 'SteamworksPy64.dll'

        else:
            # This case is theoretically unreachable
            raise UnsupportedPlatformException(f'"{platform}" is not being supported')

        library_path = os.path.join(os.getcwd(), library_file_name)
        if not os.path.isfile(library_path):
            raise MissingSteamworksLibraryException(f'Missing library at {library_path}')

        app_id_file = os.path.join(os.getcwd(), 'steam_appid.txt')
        if not os.path.isfile(app_id_file):
            raise FileNotFoundError(f'steam_appid.txt missing from {os.getcwd()}')

        with open(app_id_file, 'r') as f:
            self.app_id	= int(f.read())

        self._cdll 		= CDLL(library_path) # Throw native exception in case of error
        self._loaded 	= True

        self._load_steamworks_api()
        return self._loaded

    def _load_steamworks_api(self) -> None:
        """Load all methods from steamworks api and assign their correct arg/res types based on method map

        :return: None
        """
        if not self._loaded:
            raise SteamNotLoadedException('STEAMWORKS not yet loaded')

        for method_name, attributes in STEAMWORKS_METHODS.items():
            f = getattr(self._cdll, method_name)

            if 'restype' in attributes:
                f.restype = attributes['restype']

            if 'argtypes' in attributes:
                f.argtypes = attributes['argtypes']

            setattr(self, method_name, f)

        self._reload_steamworks_interfaces()

    def _reload_steamworks_interfaces(self) -> None:
        """Reload all interface classes

        :return: None
        """
        self.Apps           = SteamApps(self)
        self.Friends        = SteamFriends(self)
        self.Matchmaking    = SteamMatchmaking(self)
        self.Music          = SteamMusic(self)
        self.Screenshots    = SteamScreenshots(self)
        self.Users          = SteamUsers(self)
        self.UserStats      = SteamUserStats(self)
        self.Utils          = SteamUtils(self)
        self.Workshop       = SteamWorkshop(self)

    def initialize(self) -> bool:
        """Initialize Steam API connection

        :return: bool
        """
        if not self.loaded():
            raise SteamNotLoadedException('STEAMWORKS not yet loaded')

        if not self.IsSteamRunning():
            raise SteamNotRunningException('Steam is not running')

        # Boot up the Steam API
        result = self._cdll.SteamInit()
        if result == 2:
            raise SteamNotRunningException('Steam is not running')

        elif result == 3:
            raise SteamConnectionException('Not logged on or connection to Steam client could not be established')

        elif result != 0:
            raise GenericSteamException('Failed to initialize STEAMWORKS API')

        return True

    def relaunch(self, app_id: int) -> bool:
        """

        :param app_id: int
        :return: None
        """
        return self._cdll.RestartAppIfNecessary()

    def unload(self) -> None:
        """Shuts down the Steamworks API, releases pointers and frees memory.

        :return: None
        """
        self._cdll.SteamShutdown()
        self._loaded    = False
        self._cdll      = None

    def loaded(self) -> bool:
        """Is library loaded and everything populated

        :return: bool
        """
        return (self._loaded and self._cdll)

    def run_callbacks(self) -> bool:
        """Execute all callbacks

        :return: bool
        """
        if not self.loaded():
            raise SteamNotLoadedException('STEAMWORKS not yet loaded')

        self._cdll.RunCallbacks()
        return True

    def run_forever(self, base_interval: float = 1.0) -> None:
        """Loop and call Steam.run_callbacks in specified interval

        :param base_interval: float
        :return: None
        """
        while True:
            self.run_callbacks()
            time.sleep(base_interval)
<commit_msg>Update __init__.py<commit_after>"""
Copyright (c) 2016 GP Garcia, CoaguCo Industries

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
__version__ = '2.0.0'
__author__  = 'GP Garcia'

import sys, os, time
from ctypes import *
from enum import Enum

import steamworks.util 		as steamworks_util
from steamworks.enums 		import *
from steamworks.structs 	import *
from steamworks.exceptions 	import *
from steamworks.methods 	import STEAMWORKS_METHODS

from steamworks.interfaces.apps         import SteamApps
from steamworks.interfaces.friends      import SteamFriends
from steamworks.interfaces.matchmaking  import SteamMatchmaking
from steamworks.interfaces.music        import SteamMusic
from steamworks.interfaces.screenshots  import SteamScreenshots
from steamworks.interfaces.users        import SteamUsers
from steamworks.interfaces.userstats    import SteamUserStats
from steamworks.interfaces.utils        import SteamUtils
from steamworks.interfaces.workshop     import SteamWorkshop

os.environ['LD_LIBRARY_PATH'] = os.getcwd()

class STEAMWORKS(object):
    """
        Primary STEAMWORKS class used for fundamental handling of the STEAMWORKS API
    """
    _arch = steamworks_util.get_arch()
    _native_supported_platforms = ['linux', 'linux2', 'darwin', 'win32']

    def __init__(self, supported_platforms: list = []) -> None:
        self._supported_platforms = supported_platforms
        self._loaded 	= False
        self._cdll 		= None

        self.app_id 	= 0

        self._initialize()

    def _initialize(self) -> bool:
        """Initialize module by loading STEAMWORKS library

        :return: bool
        """
        platform = sys.platform
        if self._supported_platforms and platform not in self._supported_platforms:
            raise UnsupportedPlatformException(f'"{platform}" has been excluded')

        if platform not in STEAMWORKS._native_supported_platforms:
            raise UnsupportedPlatformException(f'"{platform}" is not being supported')

        library_file_name = ''
        if platform in ['linux', 'linux2']:
            library_file_name = 'SteamworksPy.so'

        elif platform == 'darwin':
            library_file_name = 'SteamworksPy.dylib'

        elif platform == 'win32':
            library_file_name = 'SteamworksPy.dll' if STEAMWORKS._arch == Arch.x86 else 'SteamworksPy64.dll'

        else:
            # This case is theoretically unreachable
            raise UnsupportedPlatformException(f'"{platform}" is not being supported')

        library_path = os.path.join(os.getcwd(), library_file_name)
        if not os.path.isfile(library_path):
            raise MissingSteamworksLibraryException(f'Missing library at {library_path}')

        app_id_file = os.path.join(os.getcwd(), 'steam_appid.txt')
        if not os.path.isfile(app_id_file):
            raise FileNotFoundError(f'steam_appid.txt missing from {os.getcwd()}')

        with open(app_id_file, 'r') as f:
            self.app_id	= int(f.read())

        self._cdll 		= CDLL(library_path) # Throw native exception in case of error
        self._loaded 	= True

        self._load_steamworks_api()
        return self._loaded

    def _load_steamworks_api(self) -> None:
        """Load all methods from steamworks api and assign their correct arg/res types based on method map

        :return: None
        """
        if not self._loaded:
            raise SteamNotLoadedException('STEAMWORKS not yet loaded')

        for method_name, attributes in STEAMWORKS_METHODS.items():
            f = getattr(self._cdll, method_name)

            if 'restype' in attributes:
                f.restype = attributes['restype']

            if 'argtypes' in attributes:
                f.argtypes = attributes['argtypes']

            setattr(self, method_name, f)

        self._reload_steamworks_interfaces()

    def _reload_steamworks_interfaces(self) -> None:
        """Reload all interface classes

        :return: None
        """
        self.Apps           = SteamApps(self)
        self.Friends        = SteamFriends(self)
        self.Matchmaking    = SteamMatchmaking(self)
        self.Music          = SteamMusic(self)
        self.Screenshots    = SteamScreenshots(self)
        self.Users          = SteamUsers(self)
        self.UserStats      = SteamUserStats(self)
        self.Utils          = SteamUtils(self)
        self.Workshop       = SteamWorkshop(self)

    def initialize(self) -> bool:
        """Initialize Steam API connection

        :return: bool
        """
        if not self.loaded():
            raise SteamNotLoadedException('STEAMWORKS not yet loaded')

        if not self.IsSteamRunning():
            raise SteamNotRunningException('Steam is not running')

        # Boot up the Steam API
        result = self._cdll.SteamInit()
        if result == 2:
            raise SteamNotRunningException('Steam is not running')

        elif result == 3:
            raise SteamConnectionException('Not logged on or connection to Steam client could not be established')

        elif result != 0:
            raise GenericSteamException('Failed to initialize STEAMWORKS API')

        return True

    def relaunch(self, app_id: int) -> bool:
        """

        :param app_id: int
        :return: None
        """
        return self._cdll.RestartAppIfNecessary()

    def unload(self) -> None:
        """Shuts down the Steamworks API, releases pointers and frees memory.

        :return: None
        """
        self._cdll.SteamShutdown()
        self._loaded    = False
        self._cdll      = None

    def loaded(self) -> bool:
        """Is library loaded and everything populated

        :return: bool
        """
        return (self._loaded and self._cdll)

    def run_callbacks(self) -> bool:
        """Execute all callbacks

        :return: bool
        """
        if not self.loaded():
            raise SteamNotLoadedException('STEAMWORKS not yet loaded')

        self._cdll.RunCallbacks()
        return True

    def run_forever(self, base_interval: float = 1.0) -> None:
        """Loop and call Steam.run_callbacks in specified interval

        :param base_interval: float
        :return: None
        """
        while True:
            self.run_callbacks()
            time.sleep(base_interval)
<|endoftext|>


<commit_before>from django import forms
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from dpaste.models import Snippet
from dpaste.highlight import LEXER_LIST_ALL, LEXER_LIST, LEXER_DEFAULT
import datetime

#===============================================================================
# Snippet Form and Handling
#===============================================================================

EXPIRE_CHOICES = (
    (3600, _(u'In one hour')),
    (3600*24*7, _(u'In one week')),
    (3600*24*30, _(u'In one month')),
    (3500*24*30*12*15, _(u'Save forever')), # 15 years, I call it forever ;)
)

EXPIRE_DEFAULT = 3600*24*30

class SnippetForm(forms.ModelForm):

    lexer = forms.ChoiceField(
        choices=LEXER_LIST,
        initial=LEXER_DEFAULT,
        label=_(u'Lexer'),
    )

    expire_options = forms.ChoiceField(
        choices=EXPIRE_CHOICES,
        initial=EXPIRE_DEFAULT,
        label=_(u'Expires'),
    )

    def __init__(self, request, *args, **kwargs):
        super(SnippetForm, self).__init__(*args, **kwargs)
        self.request = request

        try:
            if self.request.session['userprefs'].get('display_all_lexer', False):
                self.fields['lexer'].choices = LEXER_LIST_ALL
        except KeyError:
            pass

        try:
            self.fields['author'].initial = self.request.session['userprefs'].get('default_name', '')
        except KeyError:
            pass

    def save(self, parent=None, *args, **kwargs):

        # Set parent snippet
        if parent:
            self.instance.parent = parent

        # Add expire datestamp
        self.instance.expires = datetime.datetime.now() + \
            datetime.timedelta(seconds=int(self.cleaned_data['expire_options']))

        # Save snippet in the db
        super(SnippetForm, self).save(*args, **kwargs)

        # Add the snippet to the user session list
        if self.request.session.get('snippet_list', False):
            if len(self.request.session['snippet_list']) >= getattr(settings, 'MAX_SNIPPETS_PER_USER', 10):
                self.request.session['snippet_list'].pop(0)
            self.request.session['snippet_list'] += [self.instance.pk]
        else:
            self.request.session['snippet_list'] = [self.instance.pk]

        return self.request, self.instance

    class Meta:
        model = Snippet
        fields = (
            'title',
            'content',
            'author',
            'lexer',
        )

#===============================================================================
# User Settings
#===============================================================================

USERPREFS_FONT_CHOICES = [(None, _(u'Default'))] + [
    (i, i) for i in sorted((
        'Monaco',
        'Bitstream Vera Sans Mono',
        'Courier New',
        'Consolas',
    ))
]

USERPREFS_SIZES = [(None, _(u'Default'))] + [(i, '%dpx' % i) for i in range(5, 25)]

class UserSettingsForm(forms.Form):

    default_name = forms.CharField(label=_(u'Default Name'), required=False)
    display_all_lexer = forms.BooleanField(
        label=_(u'Display all lexer'),
        required=False,
        widget=forms.CheckboxInput,
        help_text=_(u'This also enables the super secret \'guess lexer\' function.'),
    )
    font_family = forms.ChoiceField(label=_(u'Font Family'), required=False, choices=USERPREFS_FONT_CHOICES)
    font_size = forms.ChoiceField(label=_(u'Font Size'), required=False, choices=USERPREFS_SIZES)
    line_height = forms.ChoiceField(label=_(u'Line Height'), required=False, choices=USERPREFS_SIZES)
<commit_msg>Ok, make 100 years. Just to be sure.<commit_after>from django import forms
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from dpaste.models import Snippet
from dpaste.highlight import LEXER_LIST_ALL, LEXER_LIST, LEXER_DEFAULT
import datetime

#===============================================================================
# Snippet Form and Handling
#===============================================================================

EXPIRE_CHOICES = (
    (3600, _(u'In one hour')),
    (3600*24*7, _(u'In one week')),
    (3600*24*30, _(u'In one month')),
    (3500*24*30*12*100, _(u'Save forever')), # 100 years, I call it forever ;)
)

EXPIRE_DEFAULT = 3600*24*30

class SnippetForm(forms.ModelForm):

    lexer = forms.ChoiceField(
        choices=LEXER_LIST,
        initial=LEXER_DEFAULT,
        label=_(u'Lexer'),
    )

    expire_options = forms.ChoiceField(
        choices=EXPIRE_CHOICES,
        initial=EXPIRE_DEFAULT,
        label=_(u'Expires'),
    )

    def __init__(self, request, *args, **kwargs):
        super(SnippetForm, self).__init__(*args, **kwargs)
        self.request = request

        try:
            if self.request.session['userprefs'].get('display_all_lexer', False):
                self.fields['lexer'].choices = LEXER_LIST_ALL
        except KeyError:
            pass

        try:
            self.fields['author'].initial = self.request.session['userprefs'].get('default_name', '')
        except KeyError:
            pass

    def save(self, parent=None, *args, **kwargs):

        # Set parent snippet
        if parent:
            self.instance.parent = parent

        # Add expire datestamp
        self.instance.expires = datetime.datetime.now() + \
            datetime.timedelta(seconds=int(self.cleaned_data['expire_options']))

        # Save snippet in the db
        super(SnippetForm, self).save(*args, **kwargs)

        # Add the snippet to the user session list
        if self.request.session.get('snippet_list', False):
            if len(self.request.session['snippet_list']) >= getattr(settings, 'MAX_SNIPPETS_PER_USER', 10):
                self.request.session['snippet_list'].pop(0)
            self.request.session['snippet_list'] += [self.instance.pk]
        else:
            self.request.session['snippet_list'] = [self.instance.pk]

        return self.request, self.instance

    class Meta:
        model = Snippet
        fields = (
            'title',
            'content',
            'author',
            'lexer',
        )

#===============================================================================
# User Settings
#===============================================================================

USERPREFS_FONT_CHOICES = [(None, _(u'Default'))] + [
    (i, i) for i in sorted((
        'Monaco',
        'Bitstream Vera Sans Mono',
        'Courier New',
        'Consolas',
    ))
]

USERPREFS_SIZES = [(None, _(u'Default'))] + [(i, '%dpx' % i) for i in range(5, 25)]

class UserSettingsForm(forms.Form):

    default_name = forms.CharField(label=_(u'Default Name'), required=False)
    display_all_lexer = forms.BooleanField(
        label=_(u'Display all lexer'),
        required=False,
        widget=forms.CheckboxInput,
        help_text=_(u'This also enables the super secret \'guess lexer\' function.'),
    )
    font_family = forms.ChoiceField(label=_(u'Font Family'), required=False, choices=USERPREFS_FONT_CHOICES)
    font_size = forms.ChoiceField(label=_(u'Font Size'), required=False, choices=USERPREFS_SIZES)
    line_height = forms.ChoiceField(label=_(u'Line Height'), required=False, choices=USERPREFS_SIZES)
<|endoftext|>


<commit_before>"""
ContactBox
"""

VERSION = (0, 1, 3)

__version__ = '.'.join((str(each) for each in VERSION[:4]))

def get_version():
    """
    Returns shorter version (digit parts only) as string.
    """
    version = '.'.join((str(each) for each in VERSION[:3]))
    if len(VERSION) > 3:
        version += str(VERSION[3])
    return version
<commit_msg>version<commit_after>"""
ContactBox
"""

VERSION = (0, 1, 4)

__version__ = '.'.join((str(each) for each in VERSION[:4]))

def get_version():
    """
    Returns shorter version (digit parts only) as string.
    """
    version = '.'.join((str(each) for each in VERSION[:3]))
    if len(VERSION) > 3:
        version += str(VERSION[3])
    return version
<|endoftext|>


<commit_before>"""
ESQuery
=======

ESQuery is a library for building elasticsearch queries in a friendly,
more readable manner.

Basic usage
-----------

There should be a file and subclass of ESQuery for each index we have.

Each method returns a new object, so you can chain calls together like
SQLAlchemy. Here's an example usage:

.. code-block:: python

    q = (FormsES()
         .domain(self.domain)
         .xmlns(self.xmlns)
         .submitted(gte=self.datespan.startdate_param,
                    lt=self.datespan.enddateparam)
         .fields(['xmlns', 'domain', 'app_id'])
         .sort('received_on', desc=False)
         .size(self.pagination.count)
         .start(self.pagination.start)
         .terms_aggregation('babies.count', 'babies_saved'))
    result = q.run()
    total_docs = result.total
    hits = result.hits

Generally useful filters and queries should be abstracted away for re-use,
but you can always add your own like so:

.. code-block:: python

    q.filter({"some_arbitrary_filter": {...}})
    q.set_query({"fancy_query": {...}})

For debugging or more helpful error messages, you can use ``query.dumps()``
and ``query.pprint()``, both of which use ``json.dumps()`` and are suitable for
pasting in to ES Head or Marvel or whatever

Filtering
---------

Filters are implemented as standalone functions, so they can be composed and
nested ``q.OR(web_users(), mobile_users())``.
Filters can be passed to the ``query.filter`` method: ``q.filter(web_users())``

There is some syntactic sugar that lets you skip this boilerplate and just
call the filter as if it were a method on the query class:  ``q.web_users()``
In order to be available for this shorthand, filters are added to the
``builtin_filters`` property of the main query class.
I know that's a bit confusing, but it seemed like the best way to make filters
available in both contexts.

Generic filters applicable to all indices are available in
``corehq.apps.es.filters``.  (But most/all can also be accessed as a query
method, if appropriate)

Filtering Specific Indices
--------------------------

There is a file for each elasticsearch index (if not, feel free to add one).
This file provides filters specific to that index, as well as an
appropriately-directed ESQuery subclass with references to these filters.

These index-specific query classes also have default filters to exclude things
like inactive users or deleted docs.
These things should nearly always be excluded, but if necessary, you can remove
these with ``remove_default_filters``.

Running against production
--------------------------
Since the ESQuery library is read-only, it's mostly safe to run against
production. You can define alternate elasticsearch hosts in your localsettings
file in the ``ELASTICSEARCH_DEBUG_HOSTS`` dictionary and pass in this host name
as the ``debug_host`` to the constructor:

.. code-block:: python

    >>> CaseES(debug_host='prod').domain('dimagi').count()
    120

Language
--------

 * es_query - the entire query, filters, query, pagination, facets
 * filters - a list of the individual filters
 * query - the query, used for searching, not filtering
 * field - a field on the document. User docs have a 'domain' field.
 * lt/gt - less/greater than
 * lte/gte - less/greater than or equal to

.. TODOs:
    sorting
    Add esquery.iter() method
"""
from collections import namedtuple
from copy import deepcopy
import json

from dimagi.utils.decorators.memoized import memoized

from corehq.elastic import ES_META, ESError, run_query, scroll_query, SIZE_LIMIT, \
    ScanResult

from . import aggregations
from . import filters
from . import queries
from .utils import values_list, flatten_field_dict

class ESQuery(object):
    """
    This query builder only outputs the following query structure::

        {
            "query": {
                "filtered": {
                    "filter": {
                        "and": [
                            <filters>
                        ]
                    },
                    "query": <query>
                }
            },
            <size, sort, other params>
        }
    """
    index = None
    _exclude_source = None
    _legacy_fields = False
    _start = None
    _size = None
    _aggregations = None
    _source = None
    default_filters = {
        "match_all": filters.match_all()
    }

    def __init__(self, index=None, debug_host=None):
        from corehq.apps.userreports.util import is_ucr_table

        self.index = index if index is not None else self.index
        if self.index not in ES_META and not is_ucr_table(self.index):
            msg = "%s is not a valid ES index.  Available options are: %s" % (
                index, ', '.join(ES_META.keys()))
            raise IndexError(msg)

        self.debug_host = debug_host
        self._default_filters = deepcopy(self.default_filters)
        self._facets = []
        self._aggregations = []
        self._source = None
        self.es_query = {"query": {
            "filtered": {
                "filter": {"and": []},
                "query": queries.match_all()
            }
        }}

    @property
    def builtin_filters(self):
        """
        A list of callables that return filters. These will all be available as
        instance methods, so you can do ``self.term(field, value)`` instead of
        ``self.filter(filters.term(field, value))``
        """
        return [
            filters.term,
            filters.OR,
            filters.AND,
            filters.NOT,
            filters.range_filter,
            filters.date_range,
            filters.missing,
            filters.exists,
            filters.empty,
            filters.non_null,
            filters.doc_id,
            filters.nested,
            filters.regexp,
        ]

    def __getattr__(self, attr):
        # This is syntactic sugar
        # If you do query.<attr> and attr isn't found as a classmethod,
        # this will look for it in self.builtin_filters.
        for fn in self.builtin_filters:
            if fn.__name__ == attr:
                def add_filter(*args, **kwargs):
                    return self.filter(fn(*args, **kwargs))
                return add_filter
        raise AttributeError("There is no builtin filter named %s" % attr)

    def __getitem__(self, sliced_or_int):
        if isinstance(sliced_or_int, (int, long)):
            start = sliced_or_int
            size = 1
        else:
            start = sliced_or_int.start or 0
            size = sliced_or_int.stop - start
        return self.start(start).size(size).run().hits

    def run(self, no_hits_with_aggs=True):
        """Actually run the query.  Returns an ESQuerySet object."""
        query = self
        if no_hits_with_aggs and query.uses_aggregations():
            query = query.size(0)
        raw = run_query(query.index, query.raw_query, debug_host=query.debug_host)
        return ESQuerySet(raw, deepcopy(query))

    def scroll(self):
        """
        Run the query against the scroll api. Returns an iterator yielding each
        document that matches the query.
        """
        result = scroll_query(self.index, self.raw_query)
        return ScanResult(
            result.count,
            (ESQuerySet.normalize_result(deepcopy(self), r) for r in result)
        )

    @property
    def _filters(self):
        return self.es_query['query']['filtered']['filter']['and']

    def exclude_source(self):
        """
        Turn off _source retrieval. Mostly useful if you just want the doc_ids
        """
        self._exclude_source = True
        return self

    def filter(self, filter):
        """
        Add the passed-in filter to the query.  All filtering goes through
        this class.
        """
        query = deepcopy(self)
        query._filters.append(filter)
        return query

    @property
    def filters(self):
        """
        Return a list of the filters used in this query, suitable if you
        want to reproduce a query with additional filtering.
        """
        return self._default_filters.values() + self._filters

    def uses_aggregations(self):
        return len(self._aggregations) > 0

    def aggregation(self, aggregation):
        """
        Add the passed-in aggregation to the query
        """
        query = deepcopy(self)
        query._aggregations.append(aggregation)
        return query

    def aggregations(self, aggregations):
        query = deepcopy(self)
        query._aggregations.extend(aggregations)
        return query

    def terms_aggregation(self, term, name, size=None):
        return self.aggregation(aggregations.TermsAggregation(name, term, size=size))

    def date_histogram(self, name, datefield, interval, timezone=None):
        return self.aggregation(aggregations.DateHistogram(name, datefield, interval, timezone=timezone))

    @property
    def _query(self):
        return self.es_query['query']['filtered']['query']

    def set_query(self, query):
        """
        Set the query.  Most stuff we want is better done with filters, but
        if you actually want Levenshtein distance or prefix querying...
        """
        es = deepcopy(self)
        es.es_query['query']['filtered']['query'] = query
        return es

    def search_string_query(self, search_string, default_fields=None):
        """Accepts a user-defined search string"""
        return self.set_query(
            queries.search_string_query(search_string, default_fields)
        )

    def _assemble(self):
        """Build out the es_query dict"""
        self._filters.extend(self._default_filters.values())
        if self._start is not None:
            self.es_query['from'] = self._start
        self.es_query['size'] = self._size if self._size is not None else SIZE_LIMIT
        if self._source:
            self.es_query['_source'] = self._source
        if self._aggregations:
            self.es_query['aggs'] = {
                agg.name: agg.assemble()
                for agg in self._aggregations
            }

    def fields(self, fields):
        """
            Restrict the fields returned from elasticsearch

            Deprecated. Use `source` instead.
            """
        self._legacy_fields = True
        return self.source(fields)

    def source(self, include, exclude=None):
        """
            Restrict the output of _source in the queryset. This can be used to return an object in a queryset
        """
        self._exclude_source = False

        source = include
        if exclude:
            source = {
                'include': include,
                'exclude': exclude
            }
        query = deepcopy(self)
        query._source = source
        return query

    def start(self, start):
        """Pagination.  Analagous to SQL offset."""
        query = deepcopy(self)
        query._start = start
        return query

    def size(self, size):
        """Restrict number of results returned.  Analagous to SQL limit."""
        query = deepcopy(self)
        query._size = size
        return query

    @property
    def raw_query(self):
        query = deepcopy(self)
        query._assemble()
        return query.es_query

    def dumps(self, pretty=False):
        """Returns the JSON query that will be sent to elasticsearch."""
        indent = 4 if pretty else None
        return json.dumps(self.raw_query, indent=indent)

    def pprint(self):
        """pretty prints the JSON query that will be sent to elasticsearch."""
        print self.dumps(pretty=True)

    def sort(self, field, desc=False, reset_sort=True):
        """Order the results by field."""
        query = deepcopy(self)
        sort_field = {
            field: {'order': 'desc' if desc else 'asc'}
        }

        if reset_sort:
            query.es_query['sort'] = [sort_field]
        else:
            if not query.es_query.get('sort'):
                query.es_query['sort'] = []
            query.es_query['sort'].append(sort_field)

        return query

    def set_sorting_block(self, sorting_block):
        """To be used with `get_sorting_block`, which interprets datatables sorting"""
        query = deepcopy(self)
        query.es_query['sort'] = sorting_block
        return query

    def remove_default_filters(self):
        """Sensible defaults are provided.  Use this if you don't want 'em"""
        query = deepcopy(self)
        query._default_filters = {"match_all": filters.match_all()}
        return query

    def remove_default_filter(self, default):
        """Remove a specific default filter by passing in its name."""
        query = deepcopy(self)
        if default in query._default_filters:
            query._default_filters.pop(default)
        if len(query._default_filters) == 0:
            query._default_filters = {"match_all": filters.match_all()}
        return query

    def values(self, *fields):
        """modeled after django's QuerySet.values"""
        if fields:
            return self.fields(fields).run().hits
        else:
            return self.run().hits

    def values_list(self, *fields, **kwargs):
        return values_list(self.fields(fields).run().hits, *fields, **kwargs)

    def count(self):
        """Performs a minimal query to get the count of matching documents"""
        return self.size(0).run().total

class ESQuerySet(object):
    """
    The object returned from ``ESQuery.run``
     * ESQuerySet.raw is the raw response from elasticsearch
     * ESQuerySet.query is the ESQuery object
    """

    def __init__(self, raw, query):
        if 'error' in raw:
            msg = ("ElasticSearch Error\n{error}\nIndex: {index}"
                   "\nQuery: {query}").format(
                       error=raw['error'],
                       index=query.index,
                       query=query.dumps(pretty=True),
                    )
            raise ESError(msg)
        self.raw = raw
        self.query = query

    @staticmethod
    def normalize_result(query, result):
        """Return the doc from an item in the query response."""
        if query._exclude_source:
            return result['_id']
        if query._legacy_fields:
            return flatten_field_dict(result, fields_property='_source')
        else:
            return result['_source']

    @property
    def raw_hits(self):
        return self.raw['hits']['hits']

    @property
    def doc_ids(self):
        """Return just the docs ids from the response."""
        return [r['_id'] for r in self.raw_hits]

    @property
    def hits(self):
        """Return the docs from the response."""
        raw_hits = self.raw_hits
        if not raw_hits and self.query._aggregations and self.query._size == 0:
            raise EsError("no hits, did you forget about no_hits_with_aggs?")

        return [self.normalize_result(self.query, r) for r in raw_hits]

    @property
    def total(self):
        """Return the total number of docs matching the query."""
        return self.raw['hits']['total']

    def aggregation(self, name):
        return self.raw['aggregations'][name]

    @property
    @memoized
    def aggregations(self):
        aggregations = self.query._aggregations
        raw = self.raw.get('aggregations', {})
        results = namedtuple('aggregation_results', [a.name for a in aggregations])
        return results(**{a.name: a.parse_result(raw) for a in aggregations})

    def __repr__(self):
        return '{}({!r}, {!r})'.format(self.__class__.__name__, self.raw, self.query)

class HQESQuery(ESQuery):
    """
    Query logic specific to CommCareHQ
    """
    @property
    def builtin_filters(self):
        return [
            filters.doc_id,
            filters.doc_type,
            filters.domain,
        ] + super(HQESQuery, self).builtin_filters
<commit_msg>cleanup error<commit_after>"""
ESQuery
=======

ESQuery is a library for building elasticsearch queries in a friendly,
more readable manner.

Basic usage
-----------

There should be a file and subclass of ESQuery for each index we have.

Each method returns a new object, so you can chain calls together like
SQLAlchemy. Here's an example usage:

.. code-block:: python

    q = (FormsES()
         .domain(self.domain)
         .xmlns(self.xmlns)
         .submitted(gte=self.datespan.startdate_param,
                    lt=self.datespan.enddateparam)
         .fields(['xmlns', 'domain', 'app_id'])
         .sort('received_on', desc=False)
         .size(self.pagination.count)
         .start(self.pagination.start)
         .terms_aggregation('babies.count', 'babies_saved'))
    result = q.run()
    total_docs = result.total
    hits = result.hits

Generally useful filters and queries should be abstracted away for re-use,
but you can always add your own like so:

.. code-block:: python

    q.filter({"some_arbitrary_filter": {...}})
    q.set_query({"fancy_query": {...}})

For debugging or more helpful error messages, you can use ``query.dumps()``
and ``query.pprint()``, both of which use ``json.dumps()`` and are suitable for
pasting in to ES Head or Marvel or whatever

Filtering
---------

Filters are implemented as standalone functions, so they can be composed and
nested ``q.OR(web_users(), mobile_users())``.
Filters can be passed to the ``query.filter`` method: ``q.filter(web_users())``

There is some syntactic sugar that lets you skip this boilerplate and just
call the filter as if it were a method on the query class:  ``q.web_users()``
In order to be available for this shorthand, filters are added to the
``builtin_filters`` property of the main query class.
I know that's a bit confusing, but it seemed like the best way to make filters
available in both contexts.

Generic filters applicable to all indices are available in
``corehq.apps.es.filters``.  (But most/all can also be accessed as a query
method, if appropriate)

Filtering Specific Indices
--------------------------

There is a file for each elasticsearch index (if not, feel free to add one).
This file provides filters specific to that index, as well as an
appropriately-directed ESQuery subclass with references to these filters.

These index-specific query classes also have default filters to exclude things
like inactive users or deleted docs.
These things should nearly always be excluded, but if necessary, you can remove
these with ``remove_default_filters``.

Running against production
--------------------------
Since the ESQuery library is read-only, it's mostly safe to run against
production. You can define alternate elasticsearch hosts in your localsettings
file in the ``ELASTICSEARCH_DEBUG_HOSTS`` dictionary and pass in this host name
as the ``debug_host`` to the constructor:

.. code-block:: python

    >>> CaseES(debug_host='prod').domain('dimagi').count()
    120

Language
--------

 * es_query - the entire query, filters, query, pagination, facets
 * filters - a list of the individual filters
 * query - the query, used for searching, not filtering
 * field - a field on the document. User docs have a 'domain' field.
 * lt/gt - less/greater than
 * lte/gte - less/greater than or equal to

.. TODOs:
    sorting
    Add esquery.iter() method
"""
from collections import namedtuple
from copy import deepcopy
import json

from dimagi.utils.decorators.memoized import memoized

from corehq.elastic import ES_META, ESError, run_query, scroll_query, SIZE_LIMIT, \
    ScanResult

from . import aggregations
from . import filters
from . import queries
from .utils import values_list, flatten_field_dict

class ESQuery(object):
    """
    This query builder only outputs the following query structure::

        {
            "query": {
                "filtered": {
                    "filter": {
                        "and": [
                            <filters>
                        ]
                    },
                    "query": <query>
                }
            },
            <size, sort, other params>
        }
    """
    index = None
    _exclude_source = None
    _legacy_fields = False
    _start = None
    _size = None
    _aggregations = None
    _source = None
    default_filters = {
        "match_all": filters.match_all()
    }

    def __init__(self, index=None, debug_host=None):
        from corehq.apps.userreports.util import is_ucr_table

        self.index = index if index is not None else self.index
        if self.index not in ES_META and not is_ucr_table(self.index):
            msg = "%s is not a valid ES index.  Available options are: %s" % (
                index, ', '.join(ES_META.keys()))
            raise IndexError(msg)

        self.debug_host = debug_host
        self._default_filters = deepcopy(self.default_filters)
        self._facets = []
        self._aggregations = []
        self._source = None
        self.es_query = {"query": {
            "filtered": {
                "filter": {"and": []},
                "query": queries.match_all()
            }
        }}

    @property
    def builtin_filters(self):
        """
        A list of callables that return filters. These will all be available as
        instance methods, so you can do ``self.term(field, value)`` instead of
        ``self.filter(filters.term(field, value))``
        """
        return [
            filters.term,
            filters.OR,
            filters.AND,
            filters.NOT,
            filters.range_filter,
            filters.date_range,
            filters.missing,
            filters.exists,
            filters.empty,
            filters.non_null,
            filters.doc_id,
            filters.nested,
            filters.regexp,
        ]

    def __getattr__(self, attr):
        # This is syntactic sugar
        # If you do query.<attr> and attr isn't found as a classmethod,
        # this will look for it in self.builtin_filters.
        for fn in self.builtin_filters:
            if fn.__name__ == attr:
                def add_filter(*args, **kwargs):
                    return self.filter(fn(*args, **kwargs))
                return add_filter
        raise AttributeError("There is no builtin filter named %s" % attr)

    def __getitem__(self, sliced_or_int):
        if isinstance(sliced_or_int, (int, long)):
            start = sliced_or_int
            size = 1
        else:
            start = sliced_or_int.start or 0
            size = sliced_or_int.stop - start
        return self.start(start).size(size).run().hits

    def run(self, no_hits_with_aggs=True):
        """Actually run the query.  Returns an ESQuerySet object."""
        query = self
        if no_hits_with_aggs and query.uses_aggregations():
            query = query.size(0)
        raw = run_query(query.index, query.raw_query, debug_host=query.debug_host)
        return ESQuerySet(raw, deepcopy(query))

    def scroll(self):
        """
        Run the query against the scroll api. Returns an iterator yielding each
        document that matches the query.
        """
        result = scroll_query(self.index, self.raw_query)
        return ScanResult(
            result.count,
            (ESQuerySet.normalize_result(deepcopy(self), r) for r in result)
        )

    @property
    def _filters(self):
        return self.es_query['query']['filtered']['filter']['and']

    def exclude_source(self):
        """
        Turn off _source retrieval. Mostly useful if you just want the doc_ids
        """
        self._exclude_source = True
        return self

    def filter(self, filter):
        """
        Add the passed-in filter to the query.  All filtering goes through
        this class.
        """
        query = deepcopy(self)
        query._filters.append(filter)
        return query

    @property
    def filters(self):
        """
        Return a list of the filters used in this query, suitable if you
        want to reproduce a query with additional filtering.
        """
        return self._default_filters.values() + self._filters

    def uses_aggregations(self):
        return len(self._aggregations) > 0

    def aggregation(self, aggregation):
        """
        Add the passed-in aggregation to the query
        """
        query = deepcopy(self)
        query._aggregations.append(aggregation)
        return query

    def aggregations(self, aggregations):
        query = deepcopy(self)
        query._aggregations.extend(aggregations)
        return query

    def terms_aggregation(self, term, name, size=None):
        return self.aggregation(aggregations.TermsAggregation(name, term, size=size))

    def date_histogram(self, name, datefield, interval, timezone=None):
        return self.aggregation(aggregations.DateHistogram(name, datefield, interval, timezone=timezone))

    @property
    def _query(self):
        return self.es_query['query']['filtered']['query']

    def set_query(self, query):
        """
        Set the query.  Most stuff we want is better done with filters, but
        if you actually want Levenshtein distance or prefix querying...
        """
        es = deepcopy(self)
        es.es_query['query']['filtered']['query'] = query
        return es

    def search_string_query(self, search_string, default_fields=None):
        """Accepts a user-defined search string"""
        return self.set_query(
            queries.search_string_query(search_string, default_fields)
        )

    def _assemble(self):
        """Build out the es_query dict"""
        self._filters.extend(self._default_filters.values())
        if self._start is not None:
            self.es_query['from'] = self._start
        self.es_query['size'] = self._size if self._size is not None else SIZE_LIMIT
        if self._source:
            self.es_query['_source'] = self._source
        if self._aggregations:
            self.es_query['aggs'] = {
                agg.name: agg.assemble()
                for agg in self._aggregations
            }

    def fields(self, fields):
        """
            Restrict the fields returned from elasticsearch

            Deprecated. Use `source` instead.
            """
        self._legacy_fields = True
        return self.source(fields)

    def source(self, include, exclude=None):
        """
            Restrict the output of _source in the queryset. This can be used to return an object in a queryset
        """
        self._exclude_source = False

        source = include
        if exclude:
            source = {
                'include': include,
                'exclude': exclude
            }
        query = deepcopy(self)
        query._source = source
        return query

    def start(self, start):
        """Pagination.  Analagous to SQL offset."""
        query = deepcopy(self)
        query._start = start
        return query

    def size(self, size):
        """Restrict number of results returned.  Analagous to SQL limit."""
        query = deepcopy(self)
        query._size = size
        return query

    @property
    def raw_query(self):
        query = deepcopy(self)
        query._assemble()
        return query.es_query

    def dumps(self, pretty=False):
        """Returns the JSON query that will be sent to elasticsearch."""
        indent = 4 if pretty else None
        return json.dumps(self.raw_query, indent=indent)

    def pprint(self):
        """pretty prints the JSON query that will be sent to elasticsearch."""
        print self.dumps(pretty=True)

    def sort(self, field, desc=False, reset_sort=True):
        """Order the results by field."""
        query = deepcopy(self)
        sort_field = {
            field: {'order': 'desc' if desc else 'asc'}
        }

        if reset_sort:
            query.es_query['sort'] = [sort_field]
        else:
            if not query.es_query.get('sort'):
                query.es_query['sort'] = []
            query.es_query['sort'].append(sort_field)

        return query

    def set_sorting_block(self, sorting_block):
        """To be used with `get_sorting_block`, which interprets datatables sorting"""
        query = deepcopy(self)
        query.es_query['sort'] = sorting_block
        return query

    def remove_default_filters(self):
        """Sensible defaults are provided.  Use this if you don't want 'em"""
        query = deepcopy(self)
        query._default_filters = {"match_all": filters.match_all()}
        return query

    def remove_default_filter(self, default):
        """Remove a specific default filter by passing in its name."""
        query = deepcopy(self)
        if default in query._default_filters:
            query._default_filters.pop(default)
        if len(query._default_filters) == 0:
            query._default_filters = {"match_all": filters.match_all()}
        return query

    def values(self, *fields):
        """modeled after django's QuerySet.values"""
        if fields:
            return self.fields(fields).run().hits
        else:
            return self.run().hits

    def values_list(self, *fields, **kwargs):
        return values_list(self.fields(fields).run().hits, *fields, **kwargs)

    def count(self):
        """Performs a minimal query to get the count of matching documents"""
        return self.size(0).run().total

class ESQuerySet(object):
    """
    The object returned from ``ESQuery.run``
     * ESQuerySet.raw is the raw response from elasticsearch
     * ESQuerySet.query is the ESQuery object
    """

    def __init__(self, raw, query):
        if 'error' in raw:
            msg = ("ElasticSearch Error\n{error}\nIndex: {index}"
                   "\nQuery: {query}").format(
                       error=raw['error'],
                       index=query.index,
                       query=query.dumps(pretty=True),
                    )
            raise ESError(msg)
        self.raw = raw
        self.query = query

    @staticmethod
    def normalize_result(query, result):
        """Return the doc from an item in the query response."""
        if query._exclude_source:
            return result['_id']
        if query._legacy_fields:
            return flatten_field_dict(result, fields_property='_source')
        else:
            return result['_source']

    @property
    def raw_hits(self):
        return self.raw['hits']['hits']

    @property
    def doc_ids(self):
        """Return just the docs ids from the response."""
        return [r['_id'] for r in self.raw_hits]

    @property
    def hits(self):
        """Return the docs from the response."""
        raw_hits = self.raw_hits
        if not raw_hits and self.query.uses_aggregations() and self.query._size == 0:
            raise ESError("no hits, did you forget about no_hits_with_aggs?")

        return [self.normalize_result(self.query, r) for r in raw_hits]

    @property
    def total(self):
        """Return the total number of docs matching the query."""
        return self.raw['hits']['total']

    def aggregation(self, name):
        return self.raw['aggregations'][name]

    @property
    @memoized
    def aggregations(self):
        aggregations = self.query._aggregations
        raw = self.raw.get('aggregations', {})
        results = namedtuple('aggregation_results', [a.name for a in aggregations])
        return results(**{a.name: a.parse_result(raw) for a in aggregations})

    def __repr__(self):
        return '{}({!r}, {!r})'.format(self.__class__.__name__, self.raw, self.query)

class HQESQuery(ESQuery):
    """
    Query logic specific to CommCareHQ
    """
    @property
    def builtin_filters(self):
        return [
            filters.doc_id,
            filters.doc_type,
            filters.domain,
        ] + super(HQESQuery, self).builtin_filters
<|endoftext|>


<commit_before>import imp, os, sys

def __init__():
	pass

def load ( moduleDirectories, pluginGlobals ):

	loadignore_file = os.path.abspath(os.path.dirname(__file__)) + '/.loadignore'

	moduleExtensions  = [ "py" ] #▒▒▒  add extensions here to extend module support   ▒▒▒#
	moduleLoader_Name = "loader"
	modulePaths       = {}
	load_ignore		  = []

	content = []
	with open(loadignore_file) as f:
	    content = f.readlines()
	load_ignore = list(map(lambda x: x.replace('\n',''), content))

	for path in sys.path:
		if path.endswith ( "Sublime Text 3" + os.sep + "Packages" ):
			packagesPath = path
			break

	for directory in moduleDirectories:
		modulePaths[directory.replace('json2apex/','')] = packagesPath + os.sep + directory

	for index in range ( 0, 2 ): #▒▒▒  loads modules twice to ensure dependencies are updated  ▒▒▒#
		for dir_name, path in modulePaths.items():
			for file in os.listdir ( path ):
				for extension in moduleExtensions:

					if file.endswith ( os.extsep + extension ):
						moduleName = os.path.basename( file )[ : - len ( os.extsep + extension ) ]
						if moduleName != moduleLoader_Name and (dir_name + '.' + moduleName) not in load_ignore:
							fileObject, file, description = imp.find_module( moduleName, [ path ] )
							pluginGlobals[ moduleName ] = imp.load_module ( moduleName, fileObject, file, description )
<commit_msg>fixed package name in loader<commit_after>import imp, os, sys

def __init__():
	pass

def load ( moduleDirectories, pluginGlobals ):

	loadignore_file = os.path.abspath(os.path.dirname(__file__)) + '/.loadignore'

	moduleExtensions  = [ "py" ] #▒▒▒  add extensions here to extend module support   ▒▒▒#
	moduleLoader_Name = "loader"
	modulePaths       = {}
	load_ignore		  = []

	content = []
	with open(loadignore_file) as f:
	    content = f.readlines()
	load_ignore = list(map(lambda x: x.replace('\n',''), content))

	for path in sys.path:
		if path.endswith ( "Sublime Text 3" + os.sep + "Packages" ):
			packagesPath = path
			break

	for directory in moduleDirectories:
		modulePaths[directory.replace('json2Apex/','')] = packagesPath + os.sep + directory

	for index in range ( 0, 2 ): #▒▒▒  loads modules twice to ensure dependencies are updated  ▒▒▒#
		for dir_name, path in modulePaths.items():
			for file in os.listdir ( path ):
				for extension in moduleExtensions:

					if file.endswith ( os.extsep + extension ):
						moduleName = os.path.basename( file )[ : - len ( os.extsep + extension ) ]
						if moduleName != moduleLoader_Name and (dir_name + '.' + moduleName) not in load_ignore:
							fileObject, file, description = imp.find_module( moduleName, [ path ] )
							pluginGlobals[ moduleName ] = imp.load_module ( moduleName, fileObject, file, description )
<|endoftext|>


<commit_before>from django.contrib import admin
from django.contrib.auth.admin import UserAdmin
from django.contrib.auth.models import User

from django_digest.models import PartialDigest, UserNonce

from .models import HQApiKey, UserHistory

class DDUserNonceAdmin(admin.ModelAdmin):
    list_display = ('user', 'nonce', 'count', 'last_used_at')

class DDPartialDigestAdmin(admin.ModelAdmin):
    list_display = ('user', 'partial_digest', 'confirmed')
    search_fields = ('login',)

admin.site.register(UserNonce, DDUserNonceAdmin)
admin.site.register(PartialDigest, DDPartialDigestAdmin)

class ApiKeyInline(admin.TabularInline):
    model = HQApiKey
    readonly_fields = ('created',)
    exclude = ('key',)
    extra = 0

    def has_add_permission(self, request, obj):
        return False

class CustomUserAdmin(UserAdmin):
    inlines = [
        ApiKeyInline,
    ]

    def has_add_permission(self, request):
        return False

admin.site.unregister(User)
admin.site.register(User, CustomUserAdmin)

class HQApiKeyAdmin(admin.ModelAdmin):
    list_display = ['user', 'name', 'created', 'domain']
    list_filter = ['created', 'domain']

    readonly_fields = ('created',)
    exclude = ('key',)

admin.site.register(HQApiKey, HQApiKeyAdmin)

class UserHistoryAdmin(admin.ModelAdmin):
    list_display = ['changed_at', 'by_domain', 'for_domain', 'user_id', 'changed_by', 'action', 'message']
    list_filter = ['by_domain', 'for_domain', 'action']
    sortable_by = []

    def has_add_permission(self, request):
        return False

    def has_change_permission(self, request, obj=None):
        return False

    def has_delete_permission(self, request, obj=None):
        return False

admin.site.register(UserHistory, UserHistoryAdmin)
<commit_msg>displays all columns<commit_after>from django.contrib import admin
from django.contrib.auth.admin import UserAdmin
from django.contrib.auth.models import User

from django_digest.models import PartialDigest, UserNonce

from .models import HQApiKey, UserHistory

class DDUserNonceAdmin(admin.ModelAdmin):
    list_display = ('user', 'nonce', 'count', 'last_used_at')

class DDPartialDigestAdmin(admin.ModelAdmin):
    list_display = ('user', 'partial_digest', 'confirmed')
    search_fields = ('login',)

admin.site.register(UserNonce, DDUserNonceAdmin)
admin.site.register(PartialDigest, DDPartialDigestAdmin)

class ApiKeyInline(admin.TabularInline):
    model = HQApiKey
    readonly_fields = ('created',)
    exclude = ('key',)
    extra = 0

    def has_add_permission(self, request, obj):
        return False

class CustomUserAdmin(UserAdmin):
    inlines = [
        ApiKeyInline,
    ]

    def has_add_permission(self, request):
        return False

admin.site.unregister(User)
admin.site.register(User, CustomUserAdmin)

class HQApiKeyAdmin(admin.ModelAdmin):
    list_display = ['user', 'name', 'created', 'domain']
    list_filter = ['created', 'domain']

    readonly_fields = ('created',)
    exclude = ('key',)

admin.site.register(HQApiKey, HQApiKeyAdmin)

class UserHistoryAdmin(admin.ModelAdmin):
    list_display = ['changed_at', 'by_domain', 'for_domain', 'user_type', 'user_id', 'changed_by', 'action',
                    'changes', 'change_messages', 'changed_via', 'user_upload_record_id']
    list_filter = ['by_domain', 'for_domain']
    sortable_by = []

    def has_add_permission(self, request):
        return False

    def has_change_permission(self, request, obj=None):
        return False

    def has_delete_permission(self, request, obj=None):
        return False

admin.site.register(UserHistory, UserHistoryAdmin)
<|endoftext|>


<commit_before>#! /usr/bin/env python
# coding=utf8

from BotModule import BotModule
from time import *
import os, sys, random
from collections import defaultdict
import random, pickle, os

class MarkovChatter(object):
	def __init__(self):
		self.markov = defaultdict(list)
		self.delimiter = "\n"

		if os.path.exists("chatter.db"):
			self.markov = pickle.load(open("chatter.db", "rb"))

	def addToBrain(self, msg, length):
		buf = [self.delimiter] * length
		for word in msg.split():
			self.markov[tuple(buf)].append(word)
			del buf[0]
			buf.append(word)
		self.markov[tuple(buf)].append(self.delimiter)
		pickle.dump(self.markov, open("chatter.db", "wa+b"), -1)

	def buildSentence(self, msg, length, max=150):
		buf = msg.split()[:length]
		if len(msg.split()) > length:
			message = buf[:]
		else:
			message = []
			for i in xrange(length):
				try:
					message.append(random.choice(self.markov[random.choice(self.markov.keys())]))
				except IndexError:
					continue
		for i in xrange(max):
			try:
				next_word = random.choice(self.markov[tuple(buf)])
			except IndexError:
				continue
			if next_word == self.delimiter:
				break
			message.append(next_word)
			del buf[0]
			buf.append(next_word)
		return ' '.join(message)

class MarkovModule(BotModule):
	def __init__(self):
		self.markov = MarkovChatter()
		return

	def command(self, nick, cmd, args, type):
		return

	def onMessage(self, type, msg):
		if type == 'public':
			if(msg.startswith(self.nick + ":")):
				self.sendPublicMessage(self.markov.buildSentence(msg[7:], 12))
			else:
				self.markov.addToBrain(msg, 12)

	def help(self, nick):
		return
<commit_msg>Changed length to 10<commit_after>#! /usr/bin/env python
# coding=utf8

from BotModule import BotModule
from time import *
import os, sys, random
from collections import defaultdict
import random, pickle, os

class MarkovChatter(object):
	def __init__(self):
		self.markov = defaultdict(list)
		self.delimiter = "\n"

		if os.path.exists("chatter.db"):
			self.markov = pickle.load(open("chatter.db", "rb"))

	def addToBrain(self, msg, length):
		buf = [self.delimiter] * length
		for word in msg.split():
			self.markov[tuple(buf)].append(word)
			del buf[0]
			buf.append(word)
		self.markov[tuple(buf)].append(self.delimiter)
		pickle.dump(self.markov, open("chatter.db", "wa+b"), -1)

	def buildSentence(self, msg, length, max=150):
		buf = msg.split()[:length]
		if len(msg.split()) > length:
			message = buf[:]
		else:
			message = []
			for i in xrange(length):
				try:
					message.append(random.choice(self.markov[random.choice(self.markov.keys())]))
				except IndexError:
					continue
		for i in xrange(max):
			try:
				next_word = random.choice(self.markov[tuple(buf)])
			except IndexError:
				continue
			if next_word == self.delimiter:
				break
			message.append(next_word)
			del buf[0]
			buf.append(next_word)
		return ' '.join(message)

class MarkovModule(BotModule):
	def __init__(self):
		self.markov = MarkovChatter()
		return

	def command(self, nick, cmd, args, type):
		return

	def onMessage(self, type, msg):
		if type == 'public':
			if(msg.startswith(self.nick + ":")):
				self.sendPublicMessage(self.markov.buildSentence(msg[7:], 10))
			else:
				self.markov.addToBrain(msg, 12)

	def help(self, nick):
		return
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright 2014-2017 Parsely, Inc.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
This module exists solely for version information so we only have to change it
in one place. Based on the suggestion `here. <http://bit.ly/16LbuJF>`_

:organization: Parsely
"""

def _safe_int(string):
    """ Simple function to convert strings into ints without dying. """
    try:
        return int(string)
    except ValueError:
        return string

__version__ = "3.15.1"
VERSION = tuple(_safe_int(x) for x in __version__.split("."))
<commit_msg>Bump version to 3.16.0<commit_after># -*- coding: utf-8 -*-

# Copyright 2014-2017 Parsely, Inc.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
This module exists solely for version information so we only have to change it
in one place. Based on the suggestion `here. <http://bit.ly/16LbuJF>`_

:organization: Parsely
"""

def _safe_int(string):
    """ Simple function to convert strings into ints without dying. """
    try:
        return int(string)
    except ValueError:
        return string

__version__ = "3.16.0"
VERSION = tuple(_safe_int(x) for x in __version__.split("."))
<|endoftext|>


<commit_before>import os
import re
import asyncio

from waterbutler import streams
from waterbutler.providers import core
from waterbutler.providers import exceptions

@core.register_provider('dropbox')
class DropboxProvider(core.BaseProvider):

    BASE_URL = 'https://api.dropbox.com/1/'
    BASE_CONTENT_URL = 'https://api-content.dropbox.com/1/'

    def __init__(self, auth, credentials, settings):
        super().__init__(auth, credentials, settings)
        self.token = self.credentials['token']
        self.folder = self.settings['folder']

    def build_content_url(self, *segments, **query):
        return core.build_url(self.BASE_CONTENT_URL, *segments, **query)

    def build_path(self, path):
        return os.path.join(self.folder, path.strip('/'))

    @property
    def default_headers(self):
        return {
            'Authorization': 'Bearer {}'.format(self.token),
        }

    def can_intra_copy(self, dest_provider):
        return type(self) == type(dest_provider)

    def can_intra_move(self, dest_provider):
        return self == dest_provider

    @asyncio.coroutine
    def intra_copy(self, dest_provider, source_options, dest_options):
        from_path = self.build_path(source_options['path'])
        to_path = self.build_path(dest_options['path'])
        if self == dest_provider:
            resp = yield from self.make_request(
                'POST',
                self.build_url('fileops', 'copy'),
                data={
                    'folder': 'auto',
                    'from_path': from_path,
                    'to_path': to_path,
                },
                expects=(200, 201),
                throws=exceptions.IntraCopyError,
            )
        else:
            from_ref_resp = yield from self.make_request(
                'GET',
                self.build_url('copy_ref', 'auto', from_path),
            )
            from_ref_data = yield from from_ref_resp.json()
            resp = yield from self.make_request(
                'POST',
                data={
                    'root': 'auto',
                    'from_copy_ref': from_ref_data['copy_ref'],
                    'to_path': to_path,
                },
                headers=dest_provider.default_headers,
                expects=(200, 201),
                throws=exceptions.IntraCopyError,
            )
        metadata = yield from resp.json()
        return DropboxMetadata(metadata, self.folder).serialized()

    @asyncio.coroutine
    def intra_move(self, dest_provider, source_options, dest_options):
        from_path = self.build_path(source_options['path'])
        to_path = self.build_path(dest_options['path'])
        resp = yield from self.make_request(
            'POST',
            self.build_url('fileops', 'move'),
            data={
                'root': 'auto',
                'from_path': from_path,
                'to_path': to_path,
            },
            expects=(200, ),
            throws=exceptions.IntraMoveError,
        )
        metadata = yield from resp.json()
        return DropboxMetadata(metadata, self.folder).serialized()

    @asyncio.coroutine
    def download(self, path, revision=None, **kwargs):
        resp = yield from self.make_request(
            'GET',
            self.build_content_url('files', 'auto', self.build_path(path)),
            expects=(200, ),
            throws=exceptions.DownloadError,
        )
        return streams.ResponseStreamReader(resp)

    @asyncio.coroutine
    def upload(self, stream, path, **kwargs):
        resp = yield from self.make_request(
            'PUT',
            self.build_content_url('files_put', 'auto', self.build_path(path)),
            headers={'Content-Length': str(stream.size)},
            data=stream,
            expects=(200, ),
            throws=exceptions.UploadError,
        )
        metadata = yield from resp.json()
        return DropboxMetadata(metadata, self.folder).serialized()

    @asyncio.coroutine
    def delete(self, path, **kwargs):
        yield from self.make_request(
            'POST',
            self.build_url('fileops', 'delete'),
            data={'root': 'auto', 'path': self.build_path(path)},
            expects=(200, ),
            throws=exceptions.DeleteError,
        )

    @asyncio.coroutine
    def metadata(self, path, **kwargs):
        response = yield from self.make_request(
            'GET',
            self.build_url('metadata', 'auto', self.build_path(path)),
            expects=(200, ),
            throws=exceptions.MetadataError
        )

        data = yield from response.json()

        if data['is_dir']:
            return [
                DropboxMetadata(item, self.folder).serialized()
                for item in data['contents']
            ]

        return DropboxMetadata(data, self.folder).serialized()

class DropboxMetadata(core.BaseMetadata):

    def __init__(self, data, root):
        stripped = re.sub('^{}/?'.format(re.escape(root)), '', data['path'])
        assert stripped != data['path'], 'Root folder not present in path'
        data['path'] = stripped
        super().__init__(data)

    @property
    def provider(self):
        return 'dropbox'

    @property
    def kind(self):
        return 'folder' if self.raw['is_dir'] else 'file'

    @property
    def name(self):
        return os.path.split(self.raw['path'])[1]

    @property
    def path(self):
        return self.raw['path']

    @property
    def size(self):
        return self.raw['bytes']

    @property
    def modified(self):
        return self.raw['modified']
<commit_msg>Add dropbox revisions<commit_after>import os
import re
import asyncio

from waterbutler import streams
from waterbutler.providers import core
from waterbutler.providers import exceptions

@core.register_provider('dropbox')
class DropboxProvider(core.BaseProvider):

    BASE_URL = 'https://api.dropbox.com/1/'
    BASE_CONTENT_URL = 'https://api-content.dropbox.com/1/'

    def __init__(self, auth, credentials, settings):
        super().__init__(auth, credentials, settings)
        self.token = self.credentials['token']
        self.folder = self.settings['folder']

    def build_content_url(self, *segments, **query):
        return core.build_url(self.BASE_CONTENT_URL, *segments, **query)

    def build_path(self, path):
        return os.path.join(self.folder, path.strip('/'))

    @property
    def default_headers(self):
        return {
            'Authorization': 'Bearer {}'.format(self.token),
        }

    def can_intra_copy(self, dest_provider):
        return type(self) == type(dest_provider)

    def can_intra_move(self, dest_provider):
        return self == dest_provider

    @asyncio.coroutine
    def intra_copy(self, dest_provider, source_options, dest_options):
        from_path = self.build_path(source_options['path'])
        to_path = self.build_path(dest_options['path'])
        if self == dest_provider:
            resp = yield from self.make_request(
                'POST',
                self.build_url('fileops', 'copy'),
                data={
                    'folder': 'auto',
                    'from_path': from_path,
                    'to_path': to_path,
                },
                expects=(200, 201),
                throws=exceptions.IntraCopyError,
            )
        else:
            from_ref_resp = yield from self.make_request(
                'GET',
                self.build_url('copy_ref', 'auto', from_path),
            )
            from_ref_data = yield from from_ref_resp.json()
            resp = yield from self.make_request(
                'POST',
                data={
                    'root': 'auto',
                    'from_copy_ref': from_ref_data['copy_ref'],
                    'to_path': to_path,
                },
                headers=dest_provider.default_headers,
                expects=(200, 201),
                throws=exceptions.IntraCopyError,
            )
        metadata = yield from resp.json()
        return DropboxMetadata(metadata, self.folder).serialized()

    @asyncio.coroutine
    def intra_move(self, dest_provider, source_options, dest_options):
        from_path = self.build_path(source_options['path'])
        to_path = self.build_path(dest_options['path'])
        resp = yield from self.make_request(
            'POST',
            self.build_url('fileops', 'move'),
            data={
                'root': 'auto',
                'from_path': from_path,
                'to_path': to_path,
            },
            expects=(200, ),
            throws=exceptions.IntraMoveError,
        )
        metadata = yield from resp.json()
        return DropboxMetadata(metadata, self.folder).serialized()

    @asyncio.coroutine
    def download(self, path, revision=None, **kwargs):
        resp = yield from self.make_request(
            'GET',
            self.build_content_url('files', 'auto', self.build_path(path)),
            expects=(200, ),
            throws=exceptions.DownloadError,
        )
        return streams.ResponseStreamReader(resp)

    @asyncio.coroutine
    def upload(self, stream, path, **kwargs):
        resp = yield from self.make_request(
            'PUT',
            self.build_content_url('files_put', 'auto', self.build_path(path)),
            headers={'Content-Length': str(stream.size)},
            data=stream,
            expects=(200, ),
            throws=exceptions.UploadError,
        )
        metadata = yield from resp.json()
        return DropboxMetadata(metadata, self.folder).serialized()

    @asyncio.coroutine
    def delete(self, path, **kwargs):
        yield from self.make_request(
            'POST',
            self.build_url('fileops', 'delete'),
            data={'root': 'auto', 'path': self.build_path(path)},
            expects=(200, ),
            throws=exceptions.DeleteError,
        )

    @asyncio.coroutine
    def metadata(self, path, **kwargs):
        response = yield from self.make_request(
            'GET',
            self.build_url('metadata', 'auto', self.build_path(path)),
            expects=(200, ),
            throws=exceptions.MetadataError
        )

        data = yield from response.json()

        if data['is_dir']:
            return [
                DropboxMetadata(item, self.folder).serialized()
                for item in data['contents']
            ]

        return DropboxMetadata(data, self.folder).serialized()

    @asyncio.coroutine
    def revisions(self, path, **kwargs):
        response = yield from self.make_request(
            'GET',
            self.build_url('revisions', 'auto', self.build_path(path)),
            expects=(200, ),
            throws=exceptions.RevisionError
        )

        data = yield from response.json()

        return [
            DropboxRevision(item).serialized()
            for item in data
        ]

class DropboxMetadata(core.BaseMetadata):

    def __init__(self, data, root):
        stripped = re.sub('^{}/?'.format(re.escape(root)), '', data['path'])
        assert stripped != data['path'], 'Root folder not present in path'
        data['path'] = stripped
        super().__init__(data)

    @property
    def provider(self):
        return 'dropbox'

    @property
    def kind(self):
        return 'folder' if self.raw['is_dir'] else 'file'

    @property
    def name(self):
        return os.path.split(self.raw['path'])[1]

    @property
    def path(self):
        return self.raw['path']

    @property
    def size(self):
        return self.raw['bytes']

    @property
    def modified(self):
        return self.raw['modified']

# TODO dates!
class DropboxRevision(core.BaseRevision):

    @property
    def provider(self):
        return 'dropbox'

    @property
    def size(self):
        return self.raw['bytes']

    @property
    def modified(self):
        return self.raw['modified']

    @property
    def revision(self):
        return self.raw['rev']

    @property
    def extra(self):
        return {
            'revisionNumber': self.raw['revision']
        }
<|endoftext|>


<commit_before>"""
This library is responsible for handling coreference resolution parsing from the XML output
"""

class Coreference():
    """
    Reflects a grouping of mentions
    """

    def __init__(self, document, element):
        """
        Constructor method
        :param document: the document handling these relationships
        :type document:class:`corenlp_xml.document.Document`
        :param element: an lxml element
        :type element:class:`lxml.etree.ElementBase`
        """
        self._element = element
        self._mentions = None
        self._representative = None
        self.document = document

    @property
    def mentions(self):
        if self._mentions is None:
            self._mentions = []
            for mention_element in self._element.xpath('mention'):
                this_mention = Mention(self, mention_element)
                self._mentions.append(this_mention)
                if this_mention.representative:
                    self._representative = this_mention
        return self._mentions

    @property
    def representative(self):
        if self._representative is None:
            """ Lazy-loaded! """
            self.mentions
        return self._representative

class Mention():
    """
    Reflects a given mention
    """

    def __init__(self, coref, element):
        """
        Constructor method
        :param coref: a coreference grouping
        :type coref:class:`corenlp_xml.coreference.Coreference`
        :param element: an xml element
        :type element:class:`lxml.etree.ElementBase`
        """
        self._coref = coref
        self._element = element
        self._start = int(element.xpath('start/text()')[0])
        self._end = int(element.xpath('end/text()')[0])
        self._sentence_id = int(element.xpath('sentence/text()')[0])
        self._sentence = None
        self._head_id = int(element.xpath('head/text()')[0])
        self._head = None
        self.text = element.xpath('text/text()')[0]

    @property
    def sentence(self):
        """
        :return: the sentence this mention relates to
        :rtype:class:`corenlp_xml.document.Sentence`
        """
        if self._sentence is None:
            sentences = self._element.xpath('sentence/text()')
            if len(sentences) > 0:
                self._sentence = self._coref.document.get_sentence_by_id(int(sentences[0]))
        return self._sentence

    @property
    def siblings(self):
        """
        :return: the other mentions for this coref group
        :rtype:list
        """
        return [mention for mention in self._coref.mentions if mention is not self]

    @property
    def tokens(self):
        """
        :return: a list of tokens relating to this sentence
        :rtype:list
        """
        return self.sentence.tokens[self._start-1:self._end-1]

    @property
    def head(self):
        """
        :return: the token corresponding to the head
        :rtype:class:`corenlp_xml.document.Token`
        """
        if self._head is None:
            self._head = self.sentence.tokens[self._head_id]
        return self._head

    @property
    def representative(self):
        """
        Interprets and normalizes the "representative" attribute"
        """
        return self._element.get('representative', False) == 'true'<commit_msg>fix rare(?) coreference text bug<commit_after>"""
This library is responsible for handling coreference resolution parsing from the XML output
"""

class Coreference():
    """
    Reflects a grouping of mentions
    """

    def __init__(self, document, element):
        """
        Constructor method
        :param document: the document handling these relationships
        :type document:class:`corenlp_xml.document.Document`
        :param element: an lxml element
        :type element:class:`lxml.etree.ElementBase`
        """
        self._element = element
        self._mentions = None
        self._representative = None
        self.document = document

    @property
    def mentions(self):
        if self._mentions is None:
            self._mentions = []
            for mention_element in self._element.xpath('mention'):
                this_mention = Mention(self, mention_element)
                self._mentions.append(this_mention)
                if this_mention.representative:
                    self._representative = this_mention
        return self._mentions

    @property
    def representative(self):
        if self._representative is None:
            """ Lazy-loaded! """
            self.mentions
        return self._representative

class Mention():
    """
    Reflects a given mention
    """

    def __init__(self, coref, element):
        """
        Constructor method
        :param coref: a coreference grouping
        :type coref:class:`corenlp_xml.coreference.Coreference`
        :param element: an xml element
        :type element:class:`lxml.etree.ElementBase`
        """
        self._coref = coref
        self._element = element
        self._start = int(element.xpath('start/text()')[0])
        self._end = int(element.xpath('end/text()')[0])
        self._sentence_id = int(element.xpath('sentence/text()')[0])
        self._sentence = None
        self._head_id = int(element.xpath('head/text()')[0])
        self._head = None
        texts = element.xpath('text/text()')
        self.text = texts[0] if len(texts) > 0 else ''  # one-off bug?

    @property
    def sentence(self):
        """
        :return: the sentence this mention relates to
        :rtype:class:`corenlp_xml.document.Sentence`
        """
        if self._sentence is None:
            sentences = self._element.xpath('sentence/text()')
            if len(sentences) > 0:
                self._sentence = self._coref.document.get_sentence_by_id(int(sentences[0]))
        return self._sentence

    @property
    def siblings(self):
        """
        :return: the other mentions for this coref group
        :rtype:list
        """
        return [mention for mention in self._coref.mentions if mention is not self]

    @property
    def tokens(self):
        """
        :return: a list of tokens relating to this sentence
        :rtype:list
        """
        return self.sentence.tokens[self._start-1:self._end-1]

    @property
    def head(self):
        """
        :return: the token corresponding to the head
        :rtype:class:`corenlp_xml.document.Token`
        """
        if self._head is None:
            self._head = self.sentence.tokens[self._head_id]
        return self._head

    @property
    def representative(self):
        """
        Interprets and normalizes the "representative" attribute"
        """
        return self._element.get('representative', False) == 'true'
<|endoftext|>