<commit_before>import datetime
import logging
import re
from collections import OrderedDict
from decimal import Decimal, InvalidOperation

import dateutil.parser
from django.conf import settings
from django.db.models import Model
from django.db.models.fields.related import ForeignKey, OneToOneField
from lxml import etree

from common.util import findnth_occurence_in_string, normalise_unicode_string
from iati.parser.exceptions import (
    FieldValidationError, IgnoredVocabularyError, NoUpdateRequired,
    ParserError, RequiredFieldError, ValidationError
)
from iati_codelists import models as codelist_models
from iati_synchroniser.models import DatasetNote

log = logging.getLogger(__name__)

class IatiParser(object):
    # default version
    VERSION = '2.03'

    def __init__(self, root):
        self.logged_functions = []
        self.hints = []
        self.errors = []
        self.parse_start_datetime = datetime.datetime.now()
        self.dataset = None
        self.publisher = None
        self.force_reparse = False
        self.default_lang = settings.DEFAULT_LANG
        # A cache to store codelist items in memory (for each element when
        # parsing).
        # During tests, if a new model with a same name is added to the
        # database, this has to be cleared:
        self.codelist_cache = {}

        # TODO: find a way to simply save in parser functions, and actually
        # commit to db on exit
        self.model_store = OrderedDict()
        self.root = root

    def check_registration_agency_validity(self, element_name, element, ref):
        reg_agency_found = False
        if ref and findnth_occurence_in_string(ref, '-', 1) > -1:
            index = findnth_occurence_in_string(ref, '-', 1)
            reg_agency = self.get_or_none(
                codelist_models.OrganisationRegistrationAgency,
                code=ref[:index]
            )
            if reg_agency:
                reg_agency_found = True

        if not reg_agency_found:
            self.append_error(
                'FieldValidationError',
                element_name,
                "ref",
                "Must be in the format {Registration Agency} - (Registration "
                "Number}",
                element.sourceline,
                ref)

    def get_or_none(self, model, *args, **kwargs):
        code = kwargs.get('code', None)
        if code:
            code = normalise_unicode_string(code)
            try:
                model_cache = self.codelist_cache[model.__name__]
            except KeyError:
                model_cache = model.objects.in_bulk()
                self.codelist_cache[model.__name__] = model_cache
            return model_cache.get(code)

        else:

            try:
                return model.objects.get(*args, **kwargs)
            except model.DoesNotExist:
                return None

    def _get_currency_or_raise(self, model_name, currency):
        """
        get default currency if not available for currency-related fields
        """
        # TODO: this does not invalidate the whole element (budget,
        # transaction, planned disbursement) while it should
        if not currency:
            currency = getattr(self.get_model('Activity'), 'default_currency')
            if not currency:
                raise RequiredFieldError(
                    model_name,
                    "currency",
                    "must specify default-currency on iati-activity or as "
                    "currency on the element itself")

        return currency

    def makeBool(self, text):
        if text == '1' or text == 'true':
            return True
        return False

    def makeBoolNone(self, text):
        if text == '1':
            return True
        elif text == '0':
            return False

        return None

    def guess_number(self, model_name, number_string):
        # first strip non numeric values, except for -.,
        decimal_string = re.sub(r'[^\d.,-]+', '', number_string)

        try:
            return Decimal(decimal_string)
        except ValueError:
            raise FieldValidationError(
                model_name,
                "value",
                "Must be decimal or integer string")
        except InvalidOperation:
            raise FieldValidationError(
                model_name,
                "value",
                "Must be decimal or integer string")

    def isInt(self, obj):
        try:
            int(obj)
            return True
        except BaseException:
            return False

    def _normalize(self, attr):
        # attr = attr.strip(' \t\n\r').replace(" ", "")
        # attr = re.sub("[/:&',.+]", "-", attr)

        # normalize for use in the API with comma-separated values
        attr = re.sub(",", "COMMA", attr)
        return attr

    def validate_date(self, unvalidated_date):

        if unvalidated_date:
            unvalidated_date = unvalidated_date.strip(' \t\n\rZ')
        else:
            return None

        # check if standard data parser works
        try:
            date = dateutil.parser.parse(unvalidated_date, ignoretz=True)
            if date.year >= 1900 and date.year <= 2100:
                return date
            else:
                return None
        except BaseException:
            raise RequiredFieldError(
                "TO DO",
                "iso-date",
                "Unspecified or invalid. Date should be of type xml:date.")

    def get_primary_name(self, element, primary_name):
        if primary_name:
            lang = element.attrib.get(
                '{http://www.w3.org/XML/1998/namespace}lang',
                self.default_lang)
            if lang == 'en':
                # FIXME: this is hardcoded!
                primary_name = element.text[:255]
        else:
            primary_name = element.text
        return primary_name

    def load_and_parse(self, root):
        self.parse_activities(root)

    def parse_activities(self, root):
        """

        """
        for e in root.getchildren():
            self.model_store = OrderedDict()
            parsed = self.parse(e)
            # only save if the activity is updated

            if parsed:
                self.save_all_models()
                self.post_save_models()

        self.post_save_file(self.dataset)

        if settings.ERROR_LOGS_ENABLED:
            self.post_save_validators(self.dataset)

            # TODO - only delete errors on activities that were updated
            self.dataset.note_count = len(self.errors)
            self.dataset.save()

            DatasetNote.objects.filter(dataset=self.dataset).delete()
            DatasetNote.objects.bulk_create(self.errors)

    def post_save_models(self):
        print("override in children")

    def post_save_file(self, dataset):
        print("override in children")

    def append_error(self, error_type, model, field, message,
                     sourceline, variable='', iati_id=None):
        if not settings.ERROR_LOGS_ENABLED:
            return

        # get iati identifier
        iati_identifier = None
        if iati_id:
            iati_identifier = iati_id
        elif self.dataset.filetype == 1:
            activity = self.get_model('Activity')
            if activity and activity.iati_identifier:
                iati_identifier = activity.iati_identifier
            # elif activity:
            #     iati_identifier = activity.id
        else:
            organisation = self.get_model('Organisation')
            if organisation and organisation.organisation_identifier:
                iati_identifier = organisation.organisation_identifier
            # elif organisation:
            #     iati_identifier = organisation.id

        if not iati_identifier and hasattr(self, 'identifier'):
            iati_identifier = self.identifier
        elif not iati_identifier:
            iati_identifier = 'no-identifier'

        if variable:
            variable = str(variable)[0:255]

        note = DatasetNote(
            dataset=self.dataset,
            iati_identifier=iati_identifier,
            model=model,
            field=field,
            message=str(message)[0:255],
            exception_type=error_type,
            line_number=sourceline,
            variable=variable
        )

        self.errors.append(note)

    def parse(self, element):
        """All of the methods from specific parser file (i. e. IATI_2_03.py)
        get called in this method
        """
        if element is None:
            return
        if type(element).__name__ != '_Element':
            return
        if element.tag == etree.Comment:
            return

        x_path = self.root.getroottree().getpath(element)
        function_name = self.generate_function_name(x_path)

        if hasattr(self, function_name)\
                and callable(getattr(self, function_name)):
            element_method = getattr(self, function_name)

            try:
                element_method(element)
            except RequiredFieldError as e:
                log.exception(e)
                self.append_error(
                    'RequiredFieldError',
                    e.model,
                    e.field,
                    e.message,
                    element.sourceline,
                    None)
                return
            except FieldValidationError as e:
                log.exception(e)
                self.append_error(
                    'FieldValidationError',
                    e.model,
                    e.field,
                    e.message,
                    element.sourceline,
                    e.variable,
                    e.iati_id)
                return
            except ValidationError as e:
                log.exception(e)
                self.append_error(
                    'FieldValidationError',
                    e.model,
                    e.field,
                    e.message,
                    element.sourceline,
                    None,
                    e.iati_id)
                return
            except IgnoredVocabularyError as e:
                # not implemented, ignore for now
                return
            except ParserError as e:
                log.exception(e)
                self.append_error(
                    'ParserError',
                    'TO DO',
                    'TO DO',
                    e.message,
                    None,
                    element.sourceline)
                return
            except NoUpdateRequired as e:
                # do nothing, go to next activity
                return
            except Exception as e:
                log.exception(e)
                return

        for e in element.getchildren():
            self.parse(e)

        return True

    def generate_function_name(self, xpath):
        function_name = xpath.replace('/', '__')
        function_name = function_name.replace('-', '_')
        function_name = self.remove_brackets(function_name)

        return function_name[2:]

    # TODO: separate these functions in their own data structure
    # - 2015-12-02
    def get_model_list(self, key):
        if key in self.model_store:
            return self.model_store[key]
        return None

    # register last seen model of this type. Is overwritten on later encounters
    def register_model(self, key, model=None):
        if isinstance(key, Model):
            model = key
            key = key.__class__.__name__

        if key in self.model_store:
            self.model_store[key].append(model)
        else:
            self.model_store[key] = [model]

    def get_model(self, key, index=-1):
        if isinstance(key, Model):
            key = key.__class__.__name__

        if key in self.model_store:
            if len(self.model_store[key]) > 0:
                return self.model_store[key][index]
        return None

    def pop_model(self, key):
        if isinstance(key, Model):
            key = key.__class__.__name__

        if key in self.model_store:
            if len(self.model_store[key]) > 0:
                return self.model_store[key].pop()
        return None

    def save_model(self, key, index=-1):
        return self.get_model(key, index).save()

    def update_related(self, model):
        """
        Currently a workaround for foreign key assignment before save
        """
        if model.__class__.__name__ in ("OrganisationNarrative", "Narrative"):
            # This is set in parser's (currently: IATI_2_03.py)
            # 'add_narrative()' method:
            model.related_object = getattr(model, '_related_object')
        for field in model._meta.fields:
            if isinstance(field, (ForeignKey, OneToOneField)):
                setattr(model, field.name, getattr(model, field.name))

    def save_all_models(self):
        # TODO: problem: assigning unsaved model to foreign key results in
        # error because field_id has not been set (see: https://git.io/fbphN)
        for model_list in self.model_store.items():
            for model in model_list[1]:
                try:
                    self.update_related(model)

                    model.save()

                except ValueError as e:
                    # TODO: check if we need to do internal logging on these
                    # value errors
                    log.exception(e)

                except Exception as e:
                    log.exception(e)

    def remove_brackets(self, function_name):
        result = ""
        flag = True
        for c in function_name:
            if c == "[":
                flag = False
            if flag:
                result += c
            if c == "]":
                flag = True
        return result
<commit_msg>Fixed related to the  Narrative model which cause empty title & description. Fixed related ContactInfo & TransactionProvider. Make this versin compatible with lower of IATI version.<commit_after>import datetime
import logging
import re
from collections import OrderedDict
from decimal import Decimal, InvalidOperation

import dateutil.parser
from django.conf import settings
from django.db.models import Model
from django.contrib.contenttypes.models import ContentType
from lxml import etree

from common.util import findnth_occurence_in_string, normalise_unicode_string
from iati.parser.exceptions import (
    FieldValidationError, IgnoredVocabularyError, NoUpdateRequired,
    ParserError, RequiredFieldError, ValidationError
)
from iati_codelists import models as codelist_models
from iati_synchroniser.models import DatasetNote

log = logging.getLogger(__name__)

class IatiParser(object):
    # default version
    VERSION = '2.03'

    def __init__(self, root):
        self.logged_functions = []
        self.hints = []
        self.errors = []
        self.parse_start_datetime = datetime.datetime.now()
        self.dataset = None
        self.publisher = None
        self.force_reparse = False
        self.default_lang = settings.DEFAULT_LANG
        # A cache to store codelist items in memory (for each element when
        # parsing).
        # During tests, if a new model with a same name is added to the
        # database, this has to be cleared:
        self.codelist_cache = {}

        # TODO: find a way to simply save in parser functions, and actually
        # commit to db on exit
        self.model_store = OrderedDict()
        self.root = root

    def check_registration_agency_validity(self, element_name, element, ref):
        reg_agency_found = False
        if ref and findnth_occurence_in_string(ref, '-', 1) > -1:
            index = findnth_occurence_in_string(ref, '-', 1)
            reg_agency = self.get_or_none(
                codelist_models.OrganisationRegistrationAgency,
                code=ref[:index]
            )
            if reg_agency:
                reg_agency_found = True

        if not reg_agency_found:
            self.append_error(
                'FieldValidationError',
                element_name,
                "ref",
                "Must be in the format {Registration Agency} - (Registration "
                "Number}",
                element.sourceline,
                ref)

    def get_or_none(self, model, *args, **kwargs):
        code = kwargs.get('code', None)
        if code:
            code = normalise_unicode_string(code)
            try:
                model_cache = self.codelist_cache[model.__name__]
            except KeyError:
                model_cache = model.objects.in_bulk()
                self.codelist_cache[model.__name__] = model_cache
            return model_cache.get(code)

        else:

            try:
                return model.objects.get(*args, **kwargs)
            except model.DoesNotExist:
                return None

    def _get_currency_or_raise(self, model_name, currency):
        """
        get default currency if not available for currency-related fields
        """
        # TODO: this does not invalidate the whole element (budget,
        # transaction, planned disbursement) while it should
        if not currency:
            currency = getattr(self.get_model('Activity'), 'default_currency')
            if not currency:
                raise RequiredFieldError(
                    model_name,
                    "currency",
                    "must specify default-currency on iati-activity or as "
                    "currency on the element itself")

        return currency

    def makeBool(self, text):
        if text == '1' or text == 'true':
            return True
        return False

    def makeBoolNone(self, text):
        if text == '1':
            return True
        elif text == '0':
            return False

        return None

    def guess_number(self, model_name, number_string):
        # first strip non numeric values, except for -.,
        decimal_string = re.sub(r'[^\d.,-]+', '', number_string)

        try:
            return Decimal(decimal_string)
        except ValueError:
            raise FieldValidationError(
                model_name,
                "value",
                "Must be decimal or integer string")
        except InvalidOperation:
            raise FieldValidationError(
                model_name,
                "value",
                "Must be decimal or integer string")

    def isInt(self, obj):
        try:
            int(obj)
            return True
        except BaseException:
            return False

    def _normalize(self, attr):
        # attr = attr.strip(' \t\n\r').replace(" ", "")
        # attr = re.sub("[/:&',.+]", "-", attr)

        # normalize for use in the API with comma-separated values
        attr = re.sub(",", "COMMA", attr)
        return attr

    def validate_date(self, unvalidated_date):

        if unvalidated_date:
            unvalidated_date = unvalidated_date.strip(' \t\n\rZ')
        else:
            return None

        # check if standard data parser works
        try:
            date = dateutil.parser.parse(unvalidated_date, ignoretz=True)
            if date.year >= 1900 and date.year <= 2100:
                return date
            else:
                return None
        except BaseException:
            raise RequiredFieldError(
                "TO DO",
                "iso-date",
                "Unspecified or invalid. Date should be of type xml:date.")

    def get_primary_name(self, element, primary_name):
        if primary_name:
            lang = element.attrib.get(
                '{http://www.w3.org/XML/1998/namespace}lang',
                self.default_lang)
            if lang == 'en':
                # FIXME: this is hardcoded!
                primary_name = element.text[:255]
        else:
            primary_name = element.text
        return primary_name

    def load_and_parse(self, root):
        self.parse_activities(root)

    def parse_activities(self, root):
        """

        """
        for e in root.getchildren():
            self.model_store = OrderedDict()
            parsed = self.parse(e)
            # only save if the activity is updated

            if parsed:
                self.save_all_models()
                self.post_save_models()

        self.post_save_file(self.dataset)

        if settings.ERROR_LOGS_ENABLED:
            self.post_save_validators(self.dataset)

            # TODO - only delete errors on activities that were updated
            self.dataset.note_count = len(self.errors)
            self.dataset.save()

            DatasetNote.objects.filter(dataset=self.dataset).delete()
            DatasetNote.objects.bulk_create(self.errors)

    def post_save_models(self):
        print("override in children")

    def post_save_file(self, dataset):
        print("override in children")

    def append_error(self, error_type, model, field, message,
                     sourceline, variable='', iati_id=None):
        if not settings.ERROR_LOGS_ENABLED:
            return

        # get iati identifier
        iati_identifier = None
        if iati_id:
            iati_identifier = iati_id
        elif self.dataset.filetype == 1:
            activity = self.get_model('Activity')
            if activity and activity.iati_identifier:
                iati_identifier = activity.iati_identifier
            # elif activity:
            #     iati_identifier = activity.id
        else:
            organisation = self.get_model('Organisation')
            if organisation and organisation.organisation_identifier:
                iati_identifier = organisation.organisation_identifier
            # elif organisation:
            #     iati_identifier = organisation.id

        if not iati_identifier and hasattr(self, 'identifier'):
            iati_identifier = self.identifier
        elif not iati_identifier:
            iati_identifier = 'no-identifier'

        if variable:
            variable = str(variable)[0:255]

        note = DatasetNote(
            dataset=self.dataset,
            iati_identifier=iati_identifier,
            model=model,
            field=field,
            message=str(message)[0:255],
            exception_type=error_type,
            line_number=sourceline,
            variable=variable
        )

        self.errors.append(note)

    def parse(self, element):
        """All of the methods from specific parser file (i. e. IATI_2_03.py)
        get called in this method
        """
        if element is None:
            return
        if type(element).__name__ != '_Element':
            return
        if element.tag == etree.Comment:
            return

        x_path = self.root.getroottree().getpath(element)
        function_name = self.generate_function_name(x_path)

        if hasattr(self, function_name)\
                and callable(getattr(self, function_name)):
            element_method = getattr(self, function_name)

            try:
                element_method(element)
            except RequiredFieldError as e:
                log.exception(e)
                self.append_error(
                    'RequiredFieldError',
                    e.model,
                    e.field,
                    e.message,
                    element.sourceline,
                    None)
                return
            except FieldValidationError as e:
                log.exception(e)
                self.append_error(
                    'FieldValidationError',
                    e.model,
                    e.field,
                    e.message,
                    element.sourceline,
                    e.variable,
                    e.iati_id)
                return
            except ValidationError as e:
                log.exception(e)
                self.append_error(
                    'FieldValidationError',
                    e.model,
                    e.field,
                    e.message,
                    element.sourceline,
                    None,
                    e.iati_id)
                return
            except IgnoredVocabularyError as e:
                # not implemented, ignore for now
                return
            except ParserError as e:
                log.exception(e)
                self.append_error(
                    'ParserError',
                    'TO DO',
                    'TO DO',
                    e.message,
                    None,
                    element.sourceline)
                return
            except NoUpdateRequired as e:
                # do nothing, go to next activity
                return
            except Exception as e:
                log.exception(e)
                return

        for e in element.getchildren():
            self.parse(e)

        return True

    def generate_function_name(self, xpath):
        function_name = xpath.replace('/', '__')
        function_name = function_name.replace('-', '_')
        function_name = self.remove_brackets(function_name)

        return function_name[2:]

    # TODO: separate these functions in their own data structure
    # - 2015-12-02
    def get_model_list(self, key):
        if key in self.model_store:
            return self.model_store[key]
        return None

    # register last seen model of this type. Is overwritten on later encounters
    def register_model(self, key, model=None):
        if isinstance(key, Model):
            model = key
            key = key.__class__.__name__

        if key in self.model_store:
            self.model_store[key].append(model)
        else:
            self.model_store[key] = [model]

    def get_model(self, key, index=-1):
        if isinstance(key, Model):
            key = key.__class__.__name__

        if key in self.model_store:
            if len(self.model_store[key]) > 0:
                return self.model_store[key][index]
        return None

    def pop_model(self, key):
        if isinstance(key, Model):
            key = key.__class__.__name__

        if key in self.model_store:
            if len(self.model_store[key]) > 0:
                return self.model_store[key].pop()
        return None

    def save_model(self, key, index=-1):
        return self.get_model(key, index).save()

    def update_related(self, model):
        # Update contact info id for the object has been related
        # to the ContactInfo model
        model_class_name = model.__class__.__name__
        contact_info = 'ContactInfo'
        if model_class_name.find(contact_info) is not -1 \
                and model_class_name != contact_info:
            model.contact_info_id = getattr(model, 'contact_info').id

        # Update the transaction provider to make it has related
        # to transaction on itself
        transaction_provider = 'TransactionProvider'
        if model_class_name.find(transaction_provider) is not -1:
            model.transaction_id = getattr(model, 'transaction').id

    def save_all_models(self):
        for model_list in self.model_store.items():
            for model in model_list[1]:
                try:
                    self.update_related(model)

                    # This is happen only on version 2.03
                    # The content type on the narrative model
                    # has an empty value
                    if self.VERSION == '2.03':
                        narrative = 'Narrative'
                        if model_list[0].find(narrative) is not -1:
                            related_name = model_list[0].replace(narrative, '')
                            model.related_content_type = \
                                ContentType.objects.get_for_model(
                                    self.model_store.get(related_name)[0])
                    model.save()

                except Exception as e:
                    log.exception(e)

        # After all objects have been saved
        # then update each model which has related to another object.
        for model_list in self.model_store.items():
            for model in model_list[1]:
                try:
                    # Updated related object of the Narrative model.
                    narrative = 'Narrative'
                    if model_list[0].find(narrative) is not -1:
                        related_name = model_list[0].replace(narrative, '')
                        model.related_object = \
                            self.model_store.get(related_name)[0]
                        model.save()

                except Exception as e:
                    log.exception(e)

    def remove_brackets(self, function_name):
        result = ""
        flag = True
        for c in function_name:
            if c == "[":
                flag = False
            if flag:
                result += c
            if c == "]":
                flag = True
        return result
<|endoftext|>


<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""
Temporal Memory implementation in Python.
"""

from collections import defaultdict
from operator import mul

from nupic.research.temporal_memory import TemporalMemory
from nupic.bindings.algorithms import Connections, ConnectionsCell

class FastTemporalMemory(TemporalMemory):
  """
  Class implementing the Temporal Memory algorithm.
  """

  def __init__(self, *args, **kwargs):
    super(FastTemporalMemory, self).__init__(*args, **kwargs)
    self.connections = Connections(self.numberOfCells())

  def burstColumns(self,
                   activeColumns,
                   predictedColumns,
                   prevActiveCells,
                   connections):
    """
    Phase 2: Burst unpredicted columns.

    Pseudocode:

      - for each unpredicted active column
        - mark all cells as active
        - mark the best matching cell as winner cell
          - (learning)
            - if it has no matching segment
              - (optimization) if there are prev winner cells
                - add a segment to it
            - mark the segment as learning

    @param activeColumns                   (set)         Indices of active columns in `t`
    @param predictedColumns                (set)         Indices of predicted columns in `t`
    @param prevActiveCells                 (set)         Indices of active cells in `t-1`
    @param connections                     (Connections) Connectivity of layer

    @return (tuple) Contains:
                      `activeCells`      (set),
                      `winnerCells`      (set),
                      `learningSegments` (set)
    """
    activeCells = set()
    winnerCells = set()
    learningSegments = set()

    unpredictedColumns = activeColumns - predictedColumns

    for column in unpredictedColumns:
      cells = self.cellsForColumn(column)
      activeCells.update(cells)

      bestSegment = connections.mostActiveSegmentForCells(
        list(cells), list(prevActiveCells), self.minThreshold)

      if bestSegment is None:
        cell = self.leastUsedCell(cells, connections)
        # TODO: (optimization) Only do this if there are prev winner cells
        bestSegment = connections.createSegment(cell)

      bestCell = ConnectionsCell(bestSegment.cell.idx)  # TODO: clean up
      winnerCells.add(bestCell)
      learningSegments.add(bestSegment)

    return activeCells, winnerCells, learningSegments

  def learnOnSegments(self,
                      prevActiveSegments,
                      learningSegments,
                      prevActiveCells,
                      winnerCells,
                      prevWinnerCells,
                      connections):
    """
    Phase 3: Perform learning by adapting segments.

    Pseudocode:

      - (learning) for each prev active or learning segment
        - if learning segment or from winner cell
          - strengthen active synapses
          - weaken inactive synapses
        - if learning segment
          - add some synapses to the segment
            - subsample from prev winner cells

    @param prevActiveSegments           (set)         Indices of active segments in `t-1`
    @param learningSegments             (set)         Indices of learning segments in `t`
    @param prevActiveCells              (set)         Indices of active cells in `t-1`
    @param winnerCells                  (set)         Indices of winner cells in `t`
    @param prevWinnerCells              (set)         Indices of winner cells in `t-1`
    @param connections                  (Connections) Connectivity of layer
    """
    # TODO: Merge this into TemporalMemory.learnOnSegments
    for segment in prevActiveSegments | learningSegments:
      isLearningSegment = segment in learningSegments
      isFromWinnerCell = segment.cell in winnerCells

      activeSynapses = self.activeSynapsesForSegment(
        segment, prevActiveCells, connections)

      if isLearningSegment or isFromWinnerCell:
        self.adaptSegment(segment, activeSynapses, connections)

      if isLearningSegment:
        n = self.maxNewSynapseCount - len(activeSynapses)

        for presynapticCell in self.pickCellsToLearnOn(n,
                                                       segment,
                                                       prevWinnerCells,
                                                       connections):
          connections.createSynapse(segment,
                                    presynapticCell,
                                    self.initialPermanence)

  def computePredictiveCells(self, activeCells, connections):
    """
    Phase 4: Compute predictive cells due to lateral input
    on distal dendrites.

    Pseudocode:

      - for each distal dendrite segment with activity >= activationThreshold
        - mark the segment as active
        - mark the cell as predictive

    Forward propagates activity from active cells to the synapses that touch
    them, to determine which synapses are active.

    @param activeCells (set)         Indices of active cells in `t`
    @param connections (Connections) Connectivity of layer

    @return (tuple) Contains:
                      `activeSegments`  (set),
                      `predictiveCells` (set)
    """
    activity = connections.computeActivity(list(activeCells),
                                           self.connectedPermanence,
                                           self.activationThreshold)
    activeSegments = set(connections.activeSegments(activity))
    predictiveCells = set(connections.activeCells(activity))

    return activeSegments, predictiveCells

  # ==============================
  # Helper functions
  # ==============================

  def columnForCell(self, cell):
    """
    Returns the index of the column that a cell belongs to.

    @param cell (int) Cell index

    @return (int) Column index
    """
    self._validateCell(cell)

    return int(cell.idx / self.cellsPerColumn)

  def cellsForColumn(self, column):
    """
    Returns the indices of cells that belong to a column.

    @param column (int) Column index

    @return (set) Cell indices
    """
    self._validateColumn(column)

    start = self.cellsPerColumn * column
    end = start + self.cellsPerColumn
    return set([ConnectionsCell(idx) for idx in range(start, end)])

  def _validateCell(self, cell):
    """
    Raises an error if cell index is invalid.

    @param cell (int) Cell index
    """
    if cell.idx >= self.numberOfCells() or cell.idx < 0:
      raise IndexError("Invalid cell")
<commit_msg>Improved comment<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""
Temporal Memory implementation in Python.
"""

from collections import defaultdict
from operator import mul

from nupic.research.temporal_memory import TemporalMemory
from nupic.bindings.algorithms import Connections, ConnectionsCell

class FastTemporalMemory(TemporalMemory):
  """
  Class implementing the Temporal Memory algorithm.
  """

  def __init__(self, *args, **kwargs):
    super(FastTemporalMemory, self).__init__(*args, **kwargs)
    self.connections = Connections(self.numberOfCells())

  def burstColumns(self,
                   activeColumns,
                   predictedColumns,
                   prevActiveCells,
                   connections):
    """
    Phase 2: Burst unpredicted columns.

    Pseudocode:

      - for each unpredicted active column
        - mark all cells as active
        - mark the best matching cell as winner cell
          - (learning)
            - if it has no matching segment
              - (optimization) if there are prev winner cells
                - add a segment to it
            - mark the segment as learning

    @param activeColumns                   (set)         Indices of active columns in `t`
    @param predictedColumns                (set)         Indices of predicted columns in `t`
    @param prevActiveCells                 (set)         Indices of active cells in `t-1`
    @param connections                     (Connections) Connectivity of layer

    @return (tuple) Contains:
                      `activeCells`      (set),
                      `winnerCells`      (set),
                      `learningSegments` (set)
    """
    activeCells = set()
    winnerCells = set()
    learningSegments = set()

    unpredictedColumns = activeColumns - predictedColumns

    for column in unpredictedColumns:
      cells = self.cellsForColumn(column)
      activeCells.update(cells)

      bestSegment = connections.mostActiveSegmentForCells(
        list(cells), list(prevActiveCells), self.minThreshold)

      if bestSegment is None:
        cell = self.leastUsedCell(cells, connections)
        # TODO: (optimization) Only do this if there are prev winner cells
        bestSegment = connections.createSegment(cell)

      # TODO: For some reason, bestSegment.cell is garbage-collected after
      # this function returns. So we have to use the below hack. Figure out
      # why and clean up.
      bestCell = ConnectionsCell(bestSegment.cell.idx)

      winnerCells.add(bestCell)
      learningSegments.add(bestSegment)

    return activeCells, winnerCells, learningSegments

  def learnOnSegments(self,
                      prevActiveSegments,
                      learningSegments,
                      prevActiveCells,
                      winnerCells,
                      prevWinnerCells,
                      connections):
    """
    Phase 3: Perform learning by adapting segments.

    Pseudocode:

      - (learning) for each prev active or learning segment
        - if learning segment or from winner cell
          - strengthen active synapses
          - weaken inactive synapses
        - if learning segment
          - add some synapses to the segment
            - subsample from prev winner cells

    @param prevActiveSegments           (set)         Indices of active segments in `t-1`
    @param learningSegments             (set)         Indices of learning segments in `t`
    @param prevActiveCells              (set)         Indices of active cells in `t-1`
    @param winnerCells                  (set)         Indices of winner cells in `t`
    @param prevWinnerCells              (set)         Indices of winner cells in `t-1`
    @param connections                  (Connections) Connectivity of layer
    """
    # TODO: Merge this into TemporalMemory.learnOnSegments
    for segment in prevActiveSegments | learningSegments:
      isLearningSegment = segment in learningSegments
      isFromWinnerCell = segment.cell in winnerCells

      activeSynapses = self.activeSynapsesForSegment(
        segment, prevActiveCells, connections)

      if isLearningSegment or isFromWinnerCell:
        self.adaptSegment(segment, activeSynapses, connections)

      if isLearningSegment:
        n = self.maxNewSynapseCount - len(activeSynapses)

        for presynapticCell in self.pickCellsToLearnOn(n,
                                                       segment,
                                                       prevWinnerCells,
                                                       connections):
          connections.createSynapse(segment,
                                    presynapticCell,
                                    self.initialPermanence)

  def computePredictiveCells(self, activeCells, connections):
    """
    Phase 4: Compute predictive cells due to lateral input
    on distal dendrites.

    Pseudocode:

      - for each distal dendrite segment with activity >= activationThreshold
        - mark the segment as active
        - mark the cell as predictive

    Forward propagates activity from active cells to the synapses that touch
    them, to determine which synapses are active.

    @param activeCells (set)         Indices of active cells in `t`
    @param connections (Connections) Connectivity of layer

    @return (tuple) Contains:
                      `activeSegments`  (set),
                      `predictiveCells` (set)
    """
    activity = connections.computeActivity(list(activeCells),
                                           self.connectedPermanence,
                                           self.activationThreshold)
    activeSegments = set(connections.activeSegments(activity))
    predictiveCells = set(connections.activeCells(activity))

    return activeSegments, predictiveCells

  # ==============================
  # Helper functions
  # ==============================

  def columnForCell(self, cell):
    """
    Returns the index of the column that a cell belongs to.

    @param cell (int) Cell index

    @return (int) Column index
    """
    self._validateCell(cell)

    return int(cell.idx / self.cellsPerColumn)

  def cellsForColumn(self, column):
    """
    Returns the indices of cells that belong to a column.

    @param column (int) Column index

    @return (set) Cell indices
    """
    self._validateColumn(column)

    start = self.cellsPerColumn * column
    end = start + self.cellsPerColumn
    return set([ConnectionsCell(idx) for idx in range(start, end)])

  def _validateCell(self, cell):
    """
    Raises an error if cell index is invalid.

    @param cell (int) Cell index
    """
    if cell.idx >= self.numberOfCells() or cell.idx < 0:
      raise IndexError("Invalid cell")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Nicole Thomas <nicole@saltstack.com>`
'''

# Import Python Libs
from __future__ import absolute_import
import os
import random
import string

# Import Salt Libs
import integration
from salt.config import cloud_providers_config

# Import Salt Testing Libs
from salttesting.helpers import ensure_in_syspath, expensiveTest

ensure_in_syspath('../../../')

def __random_name(size=6):
    '''
    Generates a radom cloud instance name
    '''
    return 'CLOUD-TEST-' + ''.join(
        random.choice(string.ascii_uppercase + string.digits)
        for x in range(size)
    )

# Create the cloud instance name to be used throughout the tests
INSTANCE_NAME = __random_name()

class EC2Test(integration.ShellCase):
    '''
    Integration tests for the EC2 cloud provider in Salt-Cloud
    '''

    @expensiveTest
    def setUp(self):
        '''
        Sets up the test requirements
        '''
        super(EC2Test, self).setUp()

        # check if appropriate cloud provider and profile files are present
        profile_str = 'ec2-config:'
        provider = 'ec2'
        providers = self.run_cloud('--list-providers')

        if profile_str not in providers:
            self.skipTest(
                'Configuration file for {0} was not found. Check {0}.conf files '
                'in tests/integration/files/conf/cloud.*.d/ to run these tests.'
                .format(provider)
            )

        # check if id, key, keyname, securitygroup, private_key, location,
        # and provider are present
        path = os.path.join(integration.FILES,
                            'conf',
                            'cloud.providers.d',
                            provider + '.conf')
        config = cloud_providers_config(path)

        id = config['ec2-config']['ec2']['id']
        key = config['ec2-config']['ec2']['key']
        keyname = config['ec2-config']['ec2']['keyname']
        sec_group = config['ec2-config']['ec2']['securitygroup']
        private_key = config['ec2-config']['ec2']['private_key']
        location = config['ec2-config']['ec2']['location']

        conf_items = [id, key, keyname, sec_group, private_key, location]
        missing_conf_item = []

        for item in conf_items:
            if item == '':
                missing_conf_item.append(item)

        if missing_conf_item:
            self.skipTest(
                'An id, key, keyname, security group, private key, and location must '
                'be provided to run these tests. One or more of these elements is '
                'missing. Check tests/integration/files/conf/cloud.providers.d/{0}.conf'
                .format(provider)
            )

    def test_instance(self):
        '''
        Tests creating and deleting an instance on EC2 (classic)
        '''

        # create the instance
        instance = self.run_cloud('-p ec2-test {0}'.format(INSTANCE_NAME))
        ret_str = '{0}:'.format(INSTANCE_NAME)

        # check if instance returned with salt installed
        try:
            self.assertIn(ret_str, instance)
        except AssertionError:
            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME))
            raise

        # delete the instance
        delete = self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME))
        ret_str = '                    shutting-down'

        # check if deletion was performed appropriately
        try:
            self.assertIn(ret_str, delete)
        except AssertionError:
            raise

    def tearDown(self):
        '''
        Clean up after tests
        '''
        query = self.run_cloud('--query')
        ret_str = '        {0}:'.format(INSTANCE_NAME)

        # if test instance is still present, delete it
        if ret_str in query:
            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME))

if __name__ == '__main__':
    from integration import run_tests
    run_tests(EC2Test)
<commit_msg>Py3 compatibility fixes<commit_after># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Nicole Thomas <nicole@saltstack.com>`
'''

# Import Python Libs
from __future__ import absolute_import
import os
import random
import string

# Import Salt Libs
import integration
from salt.config import cloud_providers_config

# Import Salt Testing Libs
from salttesting.helpers import ensure_in_syspath, expensiveTest

ensure_in_syspath('../../../')

# Import Third-Party Libs
from salt.ext.six.moves import range  # pylint: disable=import-error,redefined-builtin

def __random_name(size=6):
    '''
    Generates a radom cloud instance name
    '''
    return 'CLOUD-TEST-' + ''.join(
        random.choice(string.ascii_uppercase + string.digits)
        for x in range(size)
    )

# Create the cloud instance name to be used throughout the tests
INSTANCE_NAME = __random_name()

class EC2Test(integration.ShellCase):
    '''
    Integration tests for the EC2 cloud provider in Salt-Cloud
    '''

    @expensiveTest
    def setUp(self):
        '''
        Sets up the test requirements
        '''
        super(EC2Test, self).setUp()

        # check if appropriate cloud provider and profile files are present
        profile_str = 'ec2-config:'
        provider = 'ec2'
        providers = self.run_cloud('--list-providers')

        if profile_str not in providers:
            self.skipTest(
                'Configuration file for {0} was not found. Check {0}.conf files '
                'in tests/integration/files/conf/cloud.*.d/ to run these tests.'
                .format(provider)
            )

        # check if id, key, keyname, securitygroup, private_key, location,
        # and provider are present
        path = os.path.join(integration.FILES,
                            'conf',
                            'cloud.providers.d',
                            provider + '.conf')
        config = cloud_providers_config(path)

        id = config['ec2-config']['ec2']['id']
        key = config['ec2-config']['ec2']['key']
        keyname = config['ec2-config']['ec2']['keyname']
        sec_group = config['ec2-config']['ec2']['securitygroup']
        private_key = config['ec2-config']['ec2']['private_key']
        location = config['ec2-config']['ec2']['location']

        conf_items = [id, key, keyname, sec_group, private_key, location]
        missing_conf_item = []

        for item in conf_items:
            if item == '':
                missing_conf_item.append(item)

        if missing_conf_item:
            self.skipTest(
                'An id, key, keyname, security group, private key, and location must '
                'be provided to run these tests. One or more of these elements is '
                'missing. Check tests/integration/files/conf/cloud.providers.d/{0}.conf'
                .format(provider)
            )

    def test_instance(self):
        '''
        Tests creating and deleting an instance on EC2 (classic)
        '''

        # create the instance
        instance = self.run_cloud('-p ec2-test {0}'.format(INSTANCE_NAME))
        ret_str = '{0}:'.format(INSTANCE_NAME)

        # check if instance returned with salt installed
        try:
            self.assertIn(ret_str, instance)
        except AssertionError:
            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME))
            raise

        # delete the instance
        delete = self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME))
        ret_str = '                    shutting-down'

        # check if deletion was performed appropriately
        try:
            self.assertIn(ret_str, delete)
        except AssertionError:
            raise

    def tearDown(self):
        '''
        Clean up after tests
        '''
        query = self.run_cloud('--query')
        ret_str = '        {0}:'.format(INSTANCE_NAME)

        # if test instance is still present, delete it
        if ret_str in query:
            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME))

if __name__ == '__main__':
    from integration import run_tests
    run_tests(EC2Test)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# OpenFisca -- A versatile microsimulation software
# By: OpenFisca Team <contact@openfisca.fr>
#
# Copyright (C) 2011, 2012, 2013, 2014, 2015 OpenFisca Team
# https://github.com/openfisca
#
# This file is part of OpenFisca.
#
# OpenFisca is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# OpenFisca is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from . import holders
from .tools import empty_clone

class AbstractEntity(object):
    column_by_name = None  # Class attribute. Must be overridden by subclasses with an OrderedDict.
    count = 0
    holder_by_name = None
    key_plural = None
    key_singular = None
    index_for_person_variable_name = None  # Class attribute. Not used for persons
    is_persons_entity = False  # Class attribute
    roles_count = None  # Not used for persons
    role_for_person_variable_name = None  # Class attribute. Not used for persons
    step_size = 0
    simulation = None
    symbol = None  # Class attribute. Must be overridden by subclasses.

    def __init__(self, simulation = None):
        self.holder_by_name = {}
        if self.is_persons_entity:
            assert self.index_for_person_variable_name is None
            assert self.role_for_person_variable_name is None
        else:
            assert self.index_for_person_variable_name is not None
            assert self.role_for_person_variable_name is not None
        if simulation is not None:
            self.simulation = simulation

    def clone(self, simulation):
        """Copy the entity just enough to be able to run the simulation without modifying the original simulation."""
        new = empty_clone(self)
        new_dict = new.__dict__

        for key, value in self.__dict__.iteritems():
            if key not in ('holder_by_name', 'simulation'):
                new_dict[key] = value

        new_dict['simulation'] = simulation
        # Caution: holders must be cloned after the simulation has been set into new.
        new_dict['holder_by_name'] = {
            name: holder.clone(new)
            for name, holder in self.holder_by_name.iteritems()
            }

        return new

    def compute(self, column_name, period = None, accept_other_period = False, requested_formulas_by_period = None):
        return self.get_or_new_holder(column_name).compute(period = period, accept_other_period = accept_other_period,
            requested_formulas_by_period = requested_formulas_by_period)

    def compute_add(self, column_name, period = None, requested_formulas_by_period = None):
        return self.get_or_new_holder(column_name).compute_add(period = period,
            requested_formulas_by_period = requested_formulas_by_period)

    def compute_add_divide(self, column_name, period = None, requested_formulas_by_period = None):
        return self.get_or_new_holder(column_name).compute_add_divide(period = period,
            requested_formulas_by_period = requested_formulas_by_period)

    def compute_divide(self, column_name, period = None, requested_formulas_by_period = None):
        return self.get_or_new_holder(column_name).compute_divide(period = period,
            requested_formulas_by_period = requested_formulas_by_period)

    def get_array(self, column_name, period = None):
        return self.get_or_new_holder(column_name).get_array(period)

    def get_or_new_holder(self, column_name):
        holder = self.holder_by_name.get(column_name)
        if holder is None:
            column = self.column_by_name[column_name]
            self.holder_by_name[column_name] = holder = holders.Holder(column = column, entity = self)
            if column.formula_class is not None:
                holder.formula = column.formula_class(holder = holder)
        return holder

    def graph(self, column_name, edges, get_input_variables_and_parameters, nodes, visited):
        self.get_or_new_holder(column_name).graph(edges, get_input_variables_and_parameters, nodes, visited)

    def iter_member_persons_role_and_id(self, member):
        assert not self.is_persons_entity
        raise NotImplementedError('Method "iter_member_persons_role_and_id" is not implemented for class {}'.format(
            self.__class__.__name__))
<commit_msg>Document roles_count<commit_after># -*- coding: utf-8 -*-

# OpenFisca -- A versatile microsimulation software
# By: OpenFisca Team <contact@openfisca.fr>
#
# Copyright (C) 2011, 2012, 2013, 2014, 2015 OpenFisca Team
# https://github.com/openfisca
#
# This file is part of OpenFisca.
#
# OpenFisca is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Founœdation, either version 3 of the
# License, or (at your option) any later version.
#
# OpenFisca is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from . import holders
from .tools import empty_clone

class AbstractEntity(object):
    column_by_name = None  # Class attribute. Must be overridden by subclasses with an OrderedDict.
    count = 0
    holder_by_name = None
    key_plural = None
    key_singular = None
    index_for_person_variable_name = None  # Class attribute. Not used for persons
    is_persons_entity = False  # Class attribute

    # Maximum number of different roles in the entity.
    # Ex : If the biggest family have 7 members, roles_count = 7 for the entity family.
    # Not relevant for a person
    roles_count = None

    role_for_person_variable_name = None  # Class attribute. Not used for persons
    step_size = 0
    simulation = None
    symbol = None  # Class attribute. Must be overridden by subclasses.

    def __init__(self, simulation = None):
        self.holder_by_name = {}
        if self.is_persons_entity:
            assert self.index_for_person_variable_name is None
            assert self.role_for_person_variable_name is None
        else:
            assert self.index_for_person_variable_name is not None
            assert self.role_for_person_variable_name is not None
        if simulation is not None:
            self.simulation = simulation

    def clone(self, simulation):
        """Copy the entity just enough to be able to run the simulation without modifying the original simulation."""
        new = empty_clone(self)
        new_dict = new.__dict__

        for key, value in self.__dict__.iteritems():
            if key not in ('holder_by_name', 'simulation'):
                new_dict[key] = value

        new_dict['simulation'] = simulation
        # Caution: holders must be cloned after the simulation has been set into new.
        new_dict['holder_by_name'] = {
            name: holder.clone(new)
            for name, holder in self.holder_by_name.iteritems()
            }

        return new

    def compute(self, column_name, period = None, accept_other_period = False, requested_formulas_by_period = None):
        return self.get_or_new_holder(column_name).compute(period = period, accept_other_period = accept_other_period,
            requested_formulas_by_period = requested_formulas_by_period)

    def compute_add(self, column_name, period = None, requested_formulas_by_period = None):
        return self.get_or_new_holder(column_name).compute_add(period = period,
            requested_formulas_by_period = requested_formulas_by_period)

    def compute_add_divide(self, column_name, period = None, requested_formulas_by_period = None):
        return self.get_or_new_holder(column_name).compute_add_divide(period = period,
            requested_formulas_by_period = requested_formulas_by_period)

    def compute_divide(self, column_name, period = None, requested_formulas_by_period = None):
        return self.get_or_new_holder(column_name).compute_divide(period = period,
            requested_formulas_by_period = requested_formulas_by_period)

    def get_array(self, column_name, period = None):
        return self.get_or_new_holder(column_name).get_array(period)

    def get_or_new_holder(self, column_name):
        holder = self.holder_by_name.get(column_name)
        if holder is None:
            column = self.column_by_name[column_name]
            self.holder_by_name[column_name] = holder = holders.Holder(column = column, entity = self)
            if column.formula_class is not None:
                holder.formula = column.formula_class(holder = holder)
        return holder

    def graph(self, column_name, edges, get_input_variables_and_parameters, nodes, visited):
        self.get_or_new_holder(column_name).graph(edges, get_input_variables_and_parameters, nodes, visited)

    def iter_member_persons_role_and_id(self, member):
        assert not self.is_persons_entity
        raise NotImplementedError('Method "iter_member_persons_role_and_id" is not implemented for class {}'.format(
            self.__class__.__name__))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# EXOSIMS documentation build configuration file, created by
# sphinx-quickstart on Wed Jul 15 13:16:49 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.napoleon',
    'sphinx.ext.todo',
    'sphinx.ext.coverage',
    'sphinx.ext.mathjax',
    'sphinx.ext.ifconfig',
    'sphinx.ext.viewcode',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'EXOSIMS'
copyright = u'2015 - 2018, SIOSlab'
author = u'SIOSlab'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.4'
# The full version, including alpha/beta/rc tags.
release = '1.4.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}
html_theme_options = {'fixed_sidebar':'true'}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'EXOSIMSdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  (master_doc, 'EXOSIMS.tex', u'EXOSIMS Documentation',
   u'SIOSlab', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'exosims', u'EXOSIMS Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  (master_doc, 'EXOSIMS', u'EXOSIMS Documentation',
   author, 'EXOSIMS', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False
<commit_msg>adding relative EXOSIMS path to docs conf.py<commit_after># -*- coding: utf-8 -*-
#
# EXOSIMS documentation build configuration file, created by
# sphinx-quickstart on Wed Jul 15 13:16:49 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))
sys.path.insert(0, os.path.abspath('..'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.napoleon',
    'sphinx.ext.todo',
    'sphinx.ext.coverage',
    'sphinx.ext.mathjax',
    'sphinx.ext.ifconfig',
    'sphinx.ext.viewcode',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'EXOSIMS'
copyright = u'2015 - 2018, SIOSlab'
author = u'SIOSlab'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.4'
# The full version, including alpha/beta/rc tags.
release = '1.4.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}
html_theme_options = {'fixed_sidebar':'true'}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'EXOSIMSdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  (master_doc, 'EXOSIMS.tex', u'EXOSIMS Documentation',
   u'SIOSlab', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'exosims', u'EXOSIMS Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  (master_doc, 'EXOSIMS', u'EXOSIMS Documentation',
   author, 'EXOSIMS', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False
<|endoftext|>


<commit_before>'''
A Python Statsd implementation with some datadog special sauce.
'''

import httplib as http_client
import logging
import optparse
import random
import re
import socket
import sys
import time
import threading
import simplejson as json
from urllib import urlencode

from metrics import MetricsAggregator

# create logger
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
# create console handler and set level to debug
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
# create formatter
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
# add formatter to ch
ch.setFormatter(formatter)
logger.addHandler(ch)

logger = logging.getLogger('dogstatsd')

class Reporter(threading.Thread):
    """
    The reporter periodically sends the aggregated metrics to the
    server.
    """

    def __init__(self, interval, metrics_aggregator, api_host, api_key=None):
        threading.Thread.__init__(self)
        self.daemon = True
        self.interval = int(interval)
        self.finished = threading.Event()
        self.metrics_aggregator = metrics_aggregator
        self.flush_count = 0

        self.api_key = api_key
        self.api_host = api_host

        self.http_conn_cls = http_client.HTTPSConnection

        match = re.match('^(https?)://(.*)', api_host)

        if match:
            self.api_host = match.group(2)
            if match.group(1) == 'http':
                self.http_conn_cls = http_client.HTTPConnection

    def end(self):
        self.finished.set()

    def run(self):
        while True:
            if self.finished.is_set():
                break
            self.finished.wait(self.interval)
            self.flush()

    def flush(self):
        try:
            self.flush_count += 1
            metrics = self.metrics_aggregator.flush()
            count = len(metrics)
            if not count:
                logger.info("Flush #{0}: No metrics to flush.".format(self.flush_count))
                return
            logger.info("Flush #{0}: flushing {1} metrics".format(self.flush_count, count))
            self.submit(metrics)
        except:
            logger.exception("Error flushing metrics")

    def submit(self, metrics):

        # HACK - Copy and pasted from dogapi, because it's a bit of a pain to distribute python
        # dependencies with the agent.
        conn = self.http_conn_cls(self.api_host)
        body = json.dumps({"series" : metrics})
        headers = {'Content-Type':'application/json'}
        method = 'POST'

        params = {'api_key':self.api_key}
        url = '/api/v1/series?%s' % urlencode(params)

        start_time = time.time()
        conn.request(method, url, body, headers)

        #FIXME: add timeout handling code here

        response = conn.getresponse()
        duration = round((time.time() - start_time) * 1000.0, 4)
        logger.info("%s %s %s%s (%sms)" % (
                        response.status, method, self.api_host, url, duration))

class Server(object):
    """
    A statsd udp server.
    """

    def __init__(self, metrics_aggregator, host, port):
        self.host = host
        self.port = int(port)
        self.address = (self.host, self.port)

        self.metrics_aggregator = metrics_aggregator

        self.buffer_size = 1024
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self.socket.bind(self.address)

    def start(self):
        """ Run the server. """
        logger.info('Starting dogstatsd server on %s' % str(self.address))
        while True:
            try:
                data = self.socket.recv(self.buffer_size)
                self.metrics_aggregator.submit(data)
            except (KeyboardInterrupt, SystemExit):
                break
            except:
                logger.exception('Error receiving datagram')

def main():
    parser = optparse.OptionParser("usage: %prog [options] api_key")
    parser.add_option("-H", '--host', dest='host', default='localhost')
    parser.add_option("-p", '--port', dest='port', default='8125')
    parser.add_option("-a", '--api-host', dest='api_host', default='https://app.datadoghq.com')
    parser.add_option("-i", '--interval', dest='interval', default='10')
    options, args = parser.parse_args()

    if not len(args):
        parser.print_help()
        return sys.exit(1)

    api_key = args[0]

    # Create the aggregator (which is the point of communication between the
    # server and reporting threads.
    aggregator = MetricsAggregator()

    # Start the reporting thread.
    reporter = Reporter(options.interval, aggregator, options.api_host, api_key)
    reporter.start()

    # Start the server.
    server = Server(aggregator, options.host, options.port)
    server.start()

    # If we're here, we're done.
    logger.info("Shutting down ...")

if __name__ == '__main__':
    main()

<commit_msg>Making api_key optional.<commit_after>'''
A Python Statsd implementation with some datadog special sauce.
'''

import httplib as http_client
import logging
import optparse
import random
import re
import socket
import sys
import time
import threading
import simplejson as json
from urllib import urlencode

from metrics import MetricsAggregator

# create logger
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
# create console handler and set level to debug
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
# create formatter
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
# add formatter to ch
ch.setFormatter(formatter)
logger.addHandler(ch)

logger = logging.getLogger('dogstatsd')

class Reporter(threading.Thread):
    """
    The reporter periodically sends the aggregated metrics to the
    server.
    """

    def __init__(self, interval, metrics_aggregator, api_host, api_key=None):
        threading.Thread.__init__(self)
        self.daemon = True
        self.interval = int(interval)
        self.finished = threading.Event()
        self.metrics_aggregator = metrics_aggregator
        self.flush_count = 0

        self.api_key = api_key
        self.api_host = api_host

        self.http_conn_cls = http_client.HTTPSConnection

        match = re.match('^(https?)://(.*)', api_host)

        if match:
            self.api_host = match.group(2)
            if match.group(1) == 'http':
                self.http_conn_cls = http_client.HTTPConnection

    def end(self):
        self.finished.set()

    def run(self):
        while True:
            if self.finished.is_set():
                break
            self.finished.wait(self.interval)
            self.flush()

    def flush(self):
        try:
            self.flush_count += 1
            metrics = self.metrics_aggregator.flush()
            count = len(metrics)
            if not count:
                logger.info("Flush #{0}: No metrics to flush.".format(self.flush_count))
                return
            logger.info("Flush #{0}: flushing {1} metrics".format(self.flush_count, count))
            self.submit(metrics)
        except:
            logger.exception("Error flushing metrics")

    def submit(self, metrics):

        # HACK - Copy and pasted from dogapi, because it's a bit of a pain to distribute python
        # dependencies with the agent.
        conn = self.http_conn_cls(self.api_host)
        body = json.dumps({"series" : metrics})
        headers = {'Content-Type':'application/json'}
        method = 'POST'

        params = {}
        if self.api_key:
            params['api_key'] = self.api_key
        url = '/api/v1/series?%s' % urlencode(params)

        start_time = time.time()
        conn.request(method, url, body, headers)

        #FIXME: add timeout handling code here

        response = conn.getresponse()
        duration = round((time.time() - start_time) * 1000.0, 4)
        logger.info("%s %s %s%s (%sms)" % (
                        response.status, method, self.api_host, url, duration))

class Server(object):
    """
    A statsd udp server.
    """

    def __init__(self, metrics_aggregator, host, port):
        self.host = host
        self.port = int(port)
        self.address = (self.host, self.port)

        self.metrics_aggregator = metrics_aggregator

        self.buffer_size = 1024
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self.socket.bind(self.address)

    def start(self):
        """ Run the server. """
        logger.info('Starting dogstatsd server on %s' % str(self.address))
        while True:
            try:
                data = self.socket.recv(self.buffer_size)
                self.metrics_aggregator.submit(data)
            except (KeyboardInterrupt, SystemExit):
                break
            except:
                logger.exception('Error receiving datagram')

def main():
    parser = optparse.OptionParser("usage: %prog [options] api_key")
    parser.add_option("-H", '--host', dest='host', default='localhost')
    parser.add_option("-p", '--port', dest='port', default='8125')
    parser.add_option("-a", '--api-host', dest='api_host', default='https://app.datadoghq.com')
    parser.add_option("-k", '--api-key', dest='api_key', default=None)
    parser.add_option("-i", '--interval', dest='interval', default='10')
    options, args = parser.parse_args()

    # Create the aggregator (which is the point of communication between the
    # server and reporting threads.
    aggregator = MetricsAggregator()

    # Start the reporting thread.
    reporter = Reporter(options.interval, aggregator, options.api_host, options.api_key)
    reporter.start()

    # Start the server.
    server = Server(aggregator, options.host, options.port)
    server.start()

    # If we're here, we're done.
    logger.info("Shutting down ...")

if __name__ == '__main__':
    main()

<|endoftext|>