<commit_before>from hashlib import sha1
import httplib
import logging
import tempfile
import time
import urllib2
try:
    import json
except ImportError:
    import simplejson as json

import rdflib
import redis

from django.conf import settings

from humfrey.sparql.endpoint import Endpoint
from humfrey.update.tasks.retrieve import USER_AGENTS

logger = logging.getLogger(__name__)

class IndexUpdater(object):
    _SEND_BLOCK_SIZE = 8096

    def __init__(self):
        self.client = redis.client.Redis(**settings.REDIS_PARAMS)

    @classmethod
    def hash_result(cls, value):
        def recursive_sort(value):
            if isinstance(value, dict):
                for subvalue in value.itervalues():
                    recursive_sort(subvalue)
            elif isinstance(value, list):
                for subvalue in value:
                    recursive_sort(subvalue)
                value.sort()
        return hash(json.dumps(recursive_sort(value)))

    def update(self, index):
        for store in index.stores.all():
            self.update_for_store(index, store)

    def update_for_store(self, index, store):
        hash_key = 'humfrey:elasticsearch:indices:%s:%s' % (index.slug, store.slug)
        endpoint = Endpoint(store.query_endpoint)

        logger.debug("Performing SPARQL query.", extra={'query': index.query})
        results = endpoint.query(index.query)
        logger.debug("SPARQL server started returning results.")

        try:
            urllib2.urlopen(index.get_index_status_url(store))
            index_exists = True
        except urllib2.HTTPError, e:
            if e.code == httplib.NOT_FOUND:
                index_exists = False
                index.update_mapping = True

                request = urllib2.Request(index.get_index_url(store))
                request.get_method = lambda: 'PUT'
                urllib2.urlopen(request)
            else:
                raise

        if index.update_mapping:
            index.update_mapping = False
            if index_exists:
                request = urllib2.Request(index.get_type_url(store))
                request.get_method = lambda : 'DELETE'
                urllib2.urlopen(request)

            if index.mapping:
                request = urllib2.Request(index.get_mapping_url(store), index.mapping)
                request.get_method = lambda : 'PUT'
                urllib2.urlopen(request)

        results = self.parse_results(index, results)
        results = self.serialize_results(hash_key, results)

        # ElasticSearch can only deal with requests up to 100MiB in size,
        # so we'll write about 50MiB into each of a series of temporary files,
        # and then sent each file as a separate request to ElasticSearch.
        request_body_files, request_body_file = [], None
        for result in results:
            if request_body_file is None or request_body_file.tell() >= 52428800:
                request_body_file = tempfile.TemporaryFile()
                request_body_files.append(request_body_file)
            request_body_file.write(result)

        for request_body_file in request_body_files:
            conn = httplib.HTTPConnection(**settings.ELASTICSEARCH_SERVER)
            conn.connect()

            conn.putrequest('POST', index.get_bulk_url(store, path=True))
            conn.putheader("User-Agent", USER_AGENTS['agent'])
            conn.putheader("Content-Length", str(request_body_file.tell()))
            conn.endheaders()

            request_body_file.seek(0)

            block = request_body_file.read(self._SEND_BLOCK_SIZE)
            while block:
                conn.send(block)
                block = request_body_file.read(self._SEND_BLOCK_SIZE)

            conn.getresponse()
            conn.close()

        logger.info("ElasticSearch update complete")

    @classmethod
    def dictify(cls, groups, src):
        dst = {}
        for key, value in src.iteritems():
            # Ignore unbound fields, and take those beginning with '_' to be
            # things we don't want to appear in our results as they are e.g.
            # intermediary variables.
            if value is None or key.startswith('_'):
                continue
            if isinstance(value, rdflib.Literal):
                try:
                    value = value.toPython()
                except ValueError:
                    raise
            x = dst
            for i in key.split('_')[:-1]:
                if i not in x:
                    x[i] = {}
                x = x[i]
            x[key.rsplit('_', 1)[-1]] = value

        for group in groups:
            x = dst
            for key in group[:-1]:
                x = x.get(key, {})
            if group[-1] in x:
                val = x[group[-1]]
                x[group[-1]] = {cls.get_id(val): val}
                if isinstance(val, dict):
                    val.pop('id', None)

        id = cls.get_id(dst)
        dst.pop('id', None)
        return {id: dst}

    @classmethod
    def get_id(cls, src):
        if not isinstance(src, dict):
            return rdflib.BNode()
        if 'uri' in src:
            return src['uri']
        elif 'id' in src:
            return src['id']
        else:
            return rdflib.BNode()

    @classmethod
    def merge_dicts(cls, groups, one, two):
        immediate_groups = set(g[0] for g in groups)

        for id in two:
            if id not in one:
                one[id] = {}
            if isinstance(two[id], basestring):
                one[id] = two[id]
                continue
            for key in two[id]:
                if key in immediate_groups:
                    if key not in one[id]:
                        one[id][key] = {}
                    cls.merge_dicts(set(g[1:] for g in groups if len(g) > 1),
                                    one[id][key], two[id][key])
                elif isinstance(two[id], dict) and isinstance(two[id][key], dict):
                    if key not in one[id]:
                        one[id][key] = {'_singleton': True}
                    one[id][key].update(two[id][key])
                elif two[id][key]:
                    one[id][key] = two[id][key]
        return one

    @classmethod
    def flatten_result(cls, results):
        if results.get('_singleton'):
            for key, value in results.items():
                if not value or key in ('_singleton', 'id'):
                    del results[key]
            return results
        results = results.values()
        for result in results:
            if not isinstance(result, dict):
                continue
            for k in list(result):
                if isinstance(result[k], dict):
                    result[k] = cls.flatten_result(result[k])
                elif k == 'id':
                    del result[k]
        results[:] = filter(bool, results)
        return results

    @classmethod
    def parse_results(cls, index, results):
        groups = index.groups.split()
        groups = tuple(g.split('_') for g in groups)
        groups = tuple(sorted(groups, key=len, reverse=True))

        out = None
        current_uri = None

        for result in results:
            if result['uri'] != current_uri:
                if out:
                    yield cls.flatten_result(out)[0]
                out = {}
                current_uri = result['uri']
            result = cls.dictify(groups, result)
            cls.merge_dicts(groups, out, result)

        if out:
            yield cls.flatten_result(out)[0]

    def serialize_results(self, hash_key, results):
        client = self.client

        next_status = time.time() + 60

        result_ids = set()
        result_count = 0
        for result in results:
            result_count += 1

            result_id = sha1(result['uri'].encode('utf-8')).hexdigest()[:8]
            result_ids.add(result_id)
            result_hash = self.hash_result(result)

            cached_hash = client.hget(hash_key, result_id)
            if cached_hash == result_hash:
                continue
            client.hset(hash_key, result_id, result_hash)

            yield '{0}\n{1}\n'.format(json.dumps({'index': {'_id': result_id}}),
                                      json.dumps(result))

            if time.time() > next_status:
                logger.info("Received %d results in SPARQL resultset", result_count)
                next_status = time.time() + 60

        result_ids = set(self.client.hkeys(hash_key)) - result_ids

        logger.info("Processed %d results from SPARQL resultset", result_count)
        logger.info("Deleting %d items", len(result_ids))

        for result_id in result_ids:
            yield '{0}\n'.format(json.dumps({'delete': {'_id': result_id}}))
<commit_msg>ElasticSearch updater now sets item_count to be the number indexed across all stores.<commit_after>from hashlib import sha1
import httplib
import logging
import tempfile
import time
import urllib2
try:
    import json
except ImportError:
    import simplejson as json

import rdflib
import redis

from django.conf import settings

from humfrey.sparql.endpoint import Endpoint
from humfrey.update.tasks.retrieve import USER_AGENTS

logger = logging.getLogger(__name__)

class IndexUpdater(object):
    _SEND_BLOCK_SIZE = 8096

    def __init__(self):
        self.client = redis.client.Redis(**settings.REDIS_PARAMS)

    @classmethod
    def hash_result(cls, value):
        def recursive_sort(value):
            if isinstance(value, dict):
                for subvalue in value.itervalues():
                    recursive_sort(subvalue)
            elif isinstance(value, list):
                for subvalue in value:
                    recursive_sort(subvalue)
                value.sort()
        return hash(json.dumps(recursive_sort(value)))

    def update(self, index):
        item_count = 0
        for store in index.stores.all():
            item_count += self.update_for_store(index, store)
        index.item_count = item_count

    def update_for_store(self, index, store):
        hash_key = 'humfrey:elasticsearch:indices:%s:%s' % (index.slug, store.slug)
        endpoint = Endpoint(store.query_endpoint)

        logger.debug("Performing SPARQL query.", extra={'query': index.query})
        results = endpoint.query(index.query)
        logger.debug("SPARQL server started returning results.")

        try:
            urllib2.urlopen(index.get_index_status_url(store))
            index_exists = True
        except urllib2.HTTPError, e:
            if e.code == httplib.NOT_FOUND:
                index_exists = False
                index.update_mapping = True

                request = urllib2.Request(index.get_index_url(store))
                request.get_method = lambda: 'PUT'
                urllib2.urlopen(request)
            else:
                raise

        if index.update_mapping:
            index.update_mapping = False
            if index_exists:
                request = urllib2.Request(index.get_type_url(store))
                request.get_method = lambda : 'DELETE'
                urllib2.urlopen(request)

            if index.mapping:
                request = urllib2.Request(index.get_mapping_url(store), index.mapping)
                request.get_method = lambda : 'PUT'
                urllib2.urlopen(request)

        results = self.parse_results(index, results)
        results = self.serialize_results(hash_key, results)

        result_count = 0

        # ElasticSearch can only deal with requests up to 100MiB in size,
        # so we'll write about 50MiB into each of a series of temporary files,
        # and then sent each file as a separate request to ElasticSearch.
        request_body_files, request_body_file = [], None
        for result in results:
            if request_body_file is None or request_body_file.tell() >= 52428800:
                request_body_file = tempfile.TemporaryFile()
                request_body_files.append(request_body_file)
            request_body_file.write(result)
            result_count += 1

        for request_body_file in request_body_files:
            conn = httplib.HTTPConnection(**settings.ELASTICSEARCH_SERVER)
            conn.connect()

            conn.putrequest('POST', index.get_bulk_url(store, path=True))
            conn.putheader("User-Agent", USER_AGENTS['agent'])
            conn.putheader("Content-Length", str(request_body_file.tell()))
            conn.endheaders()

            request_body_file.seek(0)

            block = request_body_file.read(self._SEND_BLOCK_SIZE)
            while block:
                conn.send(block)
                block = request_body_file.read(self._SEND_BLOCK_SIZE)

            conn.getresponse()
            conn.close()

        logger.info("ElasticSearch update complete")

        return result_count

    @classmethod
    def dictify(cls, groups, src):
        dst = {}
        for key, value in src.iteritems():
            # Ignore unbound fields, and take those beginning with '_' to be
            # things we don't want to appear in our results as they are e.g.
            # intermediary variables.
            if value is None or key.startswith('_'):
                continue
            if isinstance(value, rdflib.Literal):
                try:
                    value = value.toPython()
                except ValueError:
                    raise
            x = dst
            for i in key.split('_')[:-1]:
                if i not in x:
                    x[i] = {}
                x = x[i]
            x[key.rsplit('_', 1)[-1]] = value

        for group in groups:
            x = dst
            for key in group[:-1]:
                x = x.get(key, {})
            if group[-1] in x:
                val = x[group[-1]]
                x[group[-1]] = {cls.get_id(val): val}
                if isinstance(val, dict):
                    val.pop('id', None)

        id = cls.get_id(dst)
        dst.pop('id', None)
        return {id: dst}

    @classmethod
    def get_id(cls, src):
        if not isinstance(src, dict):
            return rdflib.BNode()
        if 'uri' in src:
            return src['uri']
        elif 'id' in src:
            return src['id']
        else:
            return rdflib.BNode()

    @classmethod
    def merge_dicts(cls, groups, one, two):
        immediate_groups = set(g[0] for g in groups)

        for id in two:
            if id not in one:
                one[id] = {}
            if isinstance(two[id], basestring):
                one[id] = two[id]
                continue
            for key in two[id]:
                if key in immediate_groups:
                    if key not in one[id]:
                        one[id][key] = {}
                    cls.merge_dicts(set(g[1:] for g in groups if len(g) > 1),
                                    one[id][key], two[id][key])
                elif isinstance(two[id], dict) and isinstance(two[id][key], dict):
                    if key not in one[id]:
                        one[id][key] = {'_singleton': True}
                    one[id][key].update(two[id][key])
                elif two[id][key]:
                    one[id][key] = two[id][key]
        return one

    @classmethod
    def flatten_result(cls, results):
        if results.get('_singleton'):
            for key, value in results.items():
                if not value or key in ('_singleton', 'id'):
                    del results[key]
            return results
        results = results.values()
        for result in results:
            if not isinstance(result, dict):
                continue
            for k in list(result):
                if isinstance(result[k], dict):
                    result[k] = cls.flatten_result(result[k])
                elif k == 'id':
                    del result[k]
        results[:] = filter(bool, results)
        return results

    @classmethod
    def parse_results(cls, index, results):
        groups = index.groups.split()
        groups = tuple(g.split('_') for g in groups)
        groups = tuple(sorted(groups, key=len, reverse=True))

        out = None
        current_uri = None

        for result in results:
            if result['uri'] != current_uri:
                if out:
                    yield cls.flatten_result(out)[0]
                out = {}
                current_uri = result['uri']
            result = cls.dictify(groups, result)
            cls.merge_dicts(groups, out, result)

        if out:
            yield cls.flatten_result(out)[0]

    def serialize_results(self, hash_key, results):
        client = self.client

        next_status = time.time() + 60

        result_ids = set()
        result_count = 0
        for result in results:
            result_count += 1

            result_id = sha1(result['uri'].encode('utf-8')).hexdigest()[:8]
            result_ids.add(result_id)
            result_hash = self.hash_result(result)

            cached_hash = client.hget(hash_key, result_id)
            if cached_hash == result_hash:
                continue
            client.hset(hash_key, result_id, result_hash)

            yield '{0}\n{1}\n'.format(json.dumps({'index': {'_id': result_id}}),
                                      json.dumps(result))

            if time.time() > next_status:
                logger.info("Received %d results in SPARQL resultset", result_count)
                next_status = time.time() + 60

        result_ids = set(self.client.hkeys(hash_key)) - result_ids

        logger.info("Processed %d results from SPARQL resultset", result_count)
        logger.info("Deleting %d items", len(result_ids))

        for result_id in result_ids:
            yield '{0}\n'.format(json.dumps({'delete': {'_id': result_id}}))
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import argparse
import numpy as np
import sys
from utilities import writeLog
from mapTools import readNumpyZTile, saveTileAsNumpyZ
#================================================================================================================================#
'''merge3DObjects

A script for combining two 3D objects in npz files. Can be used in e.g. mesh creation.

The merged array in the npz file is S. Perhaps more flexibility on the
matter later on.

Author: Jukka-Pekka Keskinen, FMI, 2020

'''
#================================================================================================================================#
parser = argparse.ArgumentParser(prog='merge3DObjects.py',description="Merge two 3D objects within npz files.")
parser.add_argument("-f1", "--filename1", type=str, help="Name of the file containing the first object. This is placed on top of"
                    " the object in the second file.")
parser.add_argument("-f2", "--filename2", type=str, help="Name of the file containing the second object. This acts as the"
                    "background for the object in the first file.")
parser.add_argument("-fo", "--fileout", type=str, help="Name of the output file.", default='output.npz')
parser.add_argument("-mloc", "--mergeloc", nargs=3 , type=int, default=[0,0,0], help="Merge location. Indices in the second"
                    " file where the object from first file will be placed.")

args = parser.parse_args()
writeLog( parser, args )
#================================================================================================================================#

# Load files
F1=readNumpyZTile(args.filename1)
S1=F1['S']

F2=readNumpyZTile(args.filename2)
S2=F2['S']

# Calculate merge indices
ms=args.mergelocation
me=ms+np.asarray(S1.shape)
me=np.minimum(me,np.asarray(S2.shape))

# Replace values in S2 with those from S1
S2[ms[0]:me[0],ms[1]:me[1],ms[2]:me[2]]=S1

F2['S']=S2

#Save npz
saveTileAsNumpyZ(args.fileout,F2)
<commit_msg>Added the possibility to add objects in merge3DObjects.py<commit_after>#!/usr/bin/env python3

import argparse
import numpy as np
import sys
from utilities import writeLog
from mapTools import readNumpyZTile, saveTileAsNumpyZ
#================================================================================================================================#
'''merge3DObjects

A script for combining two 3D objects in npz files. Can be used in e.g. mesh creation.

The merged array in the npz file is S. Perhaps more flexibility on the
matter later on.

Author: Jukka-Pekka Keskinen, FMI, 2020

'''
#================================================================================================================================#
parser = argparse.ArgumentParser(prog='merge3DObjects.py',description="Merge two 3D objects within npz files.")
parser.add_argument("-f1", "--filename1", type=str, help="Name of the file containing the first object. This is placed on top of"
                    " the object in the second file.")
parser.add_argument("-f2", "--filename2", type=str, help="Name of the file containing the second object. This acts as the"
                    "background for the object in the first file.")
parser.add_argument("-fo", "--fileout", type=str, help="Name of the output file.", default='output.npz')
parser.add_argument("-mloc", "--mergeloc", nargs=3 , type=int, default=[0,0,0], help="Merge location. Indices in the second"
                    " file where the object from first file will be placed.")
parser.add_argument("-a", "--add", action="store_true", default=False, help="Add object in the first file on top of the second"
                    "file.")

args = parser.parse_args()
writeLog( parser, args )
#================================================================================================================================#

# Load files
F1=readNumpyZTile(args.filename1)
S1=F1['S']

F2=readNumpyZTile(args.filename2)
S2=F2['S']

# Calculate merge indices
ms=args.mergeloc
me=ms+np.asarray(S1.shape)
me=np.minimum(me,np.asarray(S2.shape))

# Combine S1 with S2
if args.add:
    S2[ms[0]:me[0],ms[1]:me[1],ms[2]:me[2]]=S2[ms[0]:me[0],ms[1]:me[1],ms[2]:me[2]]+S1
else:
    S2[ms[0]:me[0],ms[1]:me[1],ms[2]:me[2]]=S1

F2['S']=S2

#Save npz
saveTileAsNumpyZ(args.fileout,F2)
<|endoftext|>


<commit_before>from i3pystatus import Status
from i3pystatus.updates import pacman, cower

status = Status()

# show updates in pacman/aur
status.register(
    "updates",
    format=" {Pacman}/{Cower}",
    backends=[pacman.Pacman(), cower.Cower()]
)

# show clock
status.register(
    "clock",
    format=" %a %d-%m-%y  %H:%M:%S"
)

# show/change current keyboard layout
status.register(
    "xkblayout",
    format="  {name}",
    layouts=["br", "us"]
)

# show/change volume using PA
status.register(
    "pulseaudio",
    format=" {volume}%",
    format_muted=" Mute"
)

# show network speed
status.register(
    "network",
    format_up="{interface:.2}  {bytes_recv}K  {bytes_sent}K",
    format_down="{interface:.2} ",
    interface="wlp2s0"
)

# show battery status
status.register(
    "battery",
    interval=5,
    format="[{status} ]{percentage:.0f}% {remaining}",
    alert=True,
    alert_percentage=15,
    status={
        "CHR": "",
        "DPL": "",
        "DIS": "",
        "FULL": "",
    }
)

# show disk available space
status.register(
    "disk",
    path="/",
    format=" {avail}G"
)

# show available memory
status.register(
    "mem",
    format=" {avail_mem}G",
    warn_percentage=70,
    alert_percentage=90,
    divisor=1024**3
)

# show cpu usage
status.register(
    "load",
    format=" {avg1} {avg5}"
)

# show focused window title
status.register(
    "window_title",
    max_width=79
)

status.run()
<commit_msg>Added CPU temperature<commit_after>from i3pystatus import Status
from i3pystatus.updates import pacman, cower

status = Status()

# show updates in pacman/aur
status.register(
    "updates",
    format=" {Pacman}/{Cower}",
    backends=[pacman.Pacman(), cower.Cower()]
)

# show clock
status.register(
    "clock",
    format=" %a %d-%m-%y  %H:%M:%S"
)

# show/change current keyboard layout
status.register(
    "xkblayout",
    format="  {name}",
    layouts=["br", "us"]
)

# show/change volume using PA
status.register(
    "pulseaudio",
    format=" {volume}%",
    format_muted=" Mute"
)

# show network speed
status.register(
    "network",
    format_up="{interface:.2}  {bytes_recv}K  {bytes_sent}K",
    format_down="{interface:.2} ",
    interface="wlp2s0"
)

# show battery status
status.register(
    "battery",
    interval=5,
    format="[{status} ]{percentage:.0f}% {remaining}",
    alert=True,
    alert_percentage=15,
    status={
        "CHR": "",
        "DPL": "",
        "DIS": "",
        "FULL": "",
    }
)

# show disk available space
status.register(
    "disk",
    path="/",
    format=" {avail}G"
)

# show available memory
status.register(
    "mem",
    format=" {avail_mem}G",
    warn_percentage=70,
    alert_percentage=90,
    divisor=1024**3
)

# show cpu usage
status.register(
    "load",
    format=" {avg1} {avg5}"
)

status.register(
    "temp",
    file="/sys/class/thermal/thermal_zone7/temp",
    format=" {temp}°C"
)

# show focused window title
status.register(
    "window_title",
    max_width=79
)

status.run()
<|endoftext|>


<commit_before># Copyright (c) 2011, Enthought, Ltd.
# Authors: Pietro Berkes <pberkes@enthought.com>, Andrey Rzhetsky
# License: Modified BSD license (2-clause)

"""Defines an abstract class that specifies the interface of pyAnno models."""

from traits.has_traits import HasTraits
from traits.trait_types import Int
from pyanno.util import PyannoValueError, MISSING_VALUE
import numpy as np

class AbstractModel(HasTraits):
    """Abstract class defining the interface of a pyAnno model.
    """

    # number of label classes
    nclasses = Int

    # number of annotators per item
    nannotators = Int

    @staticmethod
    def create_initial_state(nclasses):
        """Factory method returning a model with random initial parameters.

        Arguments
        ---------
        nclasses : int
            Number of label classes.
        """
        raise NotImplementedError()

    def generate_annotations(self, nitems):
        """Generate a random annotation set from the model.

        Sample a random set of annotations from the probability distribution
        defined the current model parameters.

        Arguments
        ---------
        nitems : int
            Number of items to sample

        Returns
        -------
        annotations : ndarray, shape = (n_items, n_annotators)
            annotations[i,j] is the annotation of annotator j for item i
        """
        raise NotImplementedError()

    def mle(self, annotations):
        """Computes maximum likelihood estimate (MLE) of parameters.

        Estimate the model parameters from a set of observed annotations
        using maximum likelihood estimation.

        Parameters
        ----------
        annotations : ndarray, shape = (n_items, n_annotators)
            annotations[i,j] is the annotation of annotator j for item i
        """
        raise NotImplementedError()

    def map(self, annotations):
        """Computes maximum a posteriori (MAP) estimate of parameters.

        Estimate the model parameters from a set of observed annotations
        using maximum a posteriori estimation.

        Parameters
        ----------
        annotations : ndarray, shape = (n_items, n_annotators)
            annotations[i,j] is the annotation of annotator j for item i
        """
        raise NotImplementedError()

    def log_likelihood(self, annotations):
        """Compute the log likelihood of a set of annotations given the model.

        Returns log P(annotations | current model parameters).

        Parameters
        ----------
        annotations : ndarray, shape = (n_items, n_annotators)
            annotations[i,j] is the annotation of annotator j for item i

        Returns
        -------
        log_lhood : float
            log likelihood of `annotations`
        """
        raise NotImplementedError()

    def _compute_total_nsamples(self, nsamples, burn_in_samples, thin_samples):
        """Compute the total number of samples to generate in order to return
        `nsamples` samples after burn-in and thinning.
        """
        return nsamples*thin_samples + burn_in_samples

    def _post_process_samples(self, samples, burn_in_samples, thin_samples):
        """Eliminate samples, discarding the first `burn_in_samples`,
        and thinning the rest.
        """
        return samples[burn_in_samples::thin_samples,:]

    def sample_posterior_over_accuracy(self, annotations, nsamples,
                                       burn_in_samples=0, thin_samples=1):
        """Return samples from posterior over the accuracy parameters.

        Draw samples from P(accuracy parameters | data, model parameters).
        The accuracy parameters control the probability of an annotator
        reporting the correct label (the exact nature of these parameters
        varies from model to model).

        Parameters
        ----------
        annotations : ndarray, shape = (n_items, n_annotators)
            annotations[i,j] is the annotation of annotator j for item i

        nsamples : int
            Number of samples to return (i.e., burn-in and thinning samples
            are not included)

        burn_in_samples : int
            Discard the first `burn_in_samples` during the initial burn-in
            phase, where the Monte Carlo chain converges to the posterior

        thin_samples : int
            Only return one every `thin_samples` samples in order to reduce
            the auto-correlation in the sampling chain. This is called
            "thinning" in MCMC parlance.

        Returns
        -------
        samples : ndarray, shape = (n_samples, ??)
            Array of samples from the posterior distribution over parameters.
        """
        raise NotImplementedError()

    def are_annotations_compatible(self, annotations):
        """Returns True if the annotations are compatible with the model.
        """

        # The standard implementation is: valid if the number of annotators
        # is correct, if the classes are between 0 and nclasses-1,
        # and if missing values are marked with pyanno.util.MISSING_VALUE

        masked_annotations = np.ma.masked_equal(annotations, MISSING_VALUE)

        if annotations.shape[1] != self.nannotators:
            return False

        if annotations.max() >= self.nclasses:
            return False

        if masked_annotations.min() < 0:
            return False

        return True

    def _raise_if_incompatible(self, annotations):
        if not self.are_annotations_compatible(annotations):
            raise PyannoValueError('Annotations are incompatible with model '
                                   'parameters')
<commit_msg>DOC: small fix<commit_after># Copyright (c) 2011, Enthought, Ltd.
# Authors: Pietro Berkes <pberkes@enthought.com>, Andrey Rzhetsky
# License: Modified BSD license (2-clause)

"""Defines an abstract class that specifies the interface of pyAnno models."""

from traits.has_traits import HasTraits
from traits.trait_types import Int
from pyanno.util import PyannoValueError, MISSING_VALUE
import numpy as np

class AbstractModel(HasTraits):
    """Abstract class defining the interface of a pyAnno model.
    """

    # number of label classes
    nclasses = Int

    # number of annotators per item
    nannotators = Int

    @staticmethod
    def create_initial_state(nclasses):
        """Factory method returning a model with random initial parameters.

        Arguments
        ---------
        nclasses : int
            Number of label classes
        """
        raise NotImplementedError()

    def generate_annotations(self, nitems):
        """Generate a random annotation set from the model.

        Sample a random set of annotations from the probability distribution
        defined the current model parameters.

        Arguments
        ---------
        nitems : int
            Number of items to sample

        Returns
        -------
        annotations : ndarray, shape = (n_items, n_annotators)
            annotations[i,j] is the annotation of annotator j for item i
        """
        raise NotImplementedError()

    def mle(self, annotations):
        """Computes maximum likelihood estimate (MLE) of parameters.

        Estimate the model parameters from a set of observed annotations
        using maximum likelihood estimation.

        Parameters
        ----------
        annotations : ndarray, shape = (n_items, n_annotators)
            annotations[i,j] is the annotation of annotator j for item i
        """
        raise NotImplementedError()

    def map(self, annotations):
        """Computes maximum a posteriori (MAP) estimate of parameters.

        Estimate the model parameters from a set of observed annotations
        using maximum a posteriori estimation.

        Parameters
        ----------
        annotations : ndarray, shape = (n_items, n_annotators)
            annotations[i,j] is the annotation of annotator j for item i
        """
        raise NotImplementedError()

    def log_likelihood(self, annotations):
        """Compute the log likelihood of a set of annotations given the model.

        Returns log P(annotations | current model parameters).

        Parameters
        ----------
        annotations : ndarray, shape = (n_items, n_annotators)
            annotations[i,j] is the annotation of annotator j for item i

        Returns
        -------
        log_lhood : float
            log likelihood of `annotations`
        """
        raise NotImplementedError()

    def _compute_total_nsamples(self, nsamples, burn_in_samples, thin_samples):
        """Compute the total number of samples to generate in order to return
        `nsamples` samples after burn-in and thinning.
        """
        return nsamples*thin_samples + burn_in_samples

    def _post_process_samples(self, samples, burn_in_samples, thin_samples):
        """Eliminate samples, discarding the first `burn_in_samples`,
        and thinning the rest.
        """
        return samples[burn_in_samples::thin_samples,:]

    def sample_posterior_over_accuracy(self, annotations, nsamples,
                                       burn_in_samples=0, thin_samples=1):
        """Return samples from posterior over the accuracy parameters.

        Draw samples from P(accuracy parameters | data, model parameters).
        The accuracy parameters control the probability of an annotator
        reporting the correct label (the exact nature of these parameters
        varies from model to model).

        Parameters
        ----------
        annotations : ndarray, shape = (n_items, n_annotators)
            annotations[i,j] is the annotation of annotator j for item i

        nsamples : int
            Number of samples to return (i.e., burn-in and thinning samples
            are not included)

        burn_in_samples : int
            Discard the first `burn_in_samples` during the initial burn-in
            phase, where the Monte Carlo chain converges to the posterior

        thin_samples : int
            Only return one every `thin_samples` samples in order to reduce
            the auto-correlation in the sampling chain. This is called
            "thinning" in MCMC parlance.

        Returns
        -------
        samples : ndarray, shape = (n_samples, ??)
            Array of samples from the posterior distribution over parameters.
        """
        raise NotImplementedError()

    def are_annotations_compatible(self, annotations):
        """Returns True if the annotations are compatible with the model.
        """

        # The standard implementation is: valid if the number of annotators
        # is correct, if the classes are between 0 and nclasses-1,
        # and if missing values are marked with pyanno.util.MISSING_VALUE

        masked_annotations = np.ma.masked_equal(annotations, MISSING_VALUE)

        if annotations.shape[1] != self.nannotators:
            return False

        if annotations.max() >= self.nclasses:
            return False

        if masked_annotations.min() < 0:
            return False

        return True

    def _raise_if_incompatible(self, annotations):
        if not self.are_annotations_compatible(annotations):
            raise PyannoValueError('Annotations are incompatible with model '
                                   'parameters')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
=============================================
Poisson Exponential Distribution
=============================================

This is an example demonstrating how pyglmnet with
poisson exponential distribution works.

"""

########################################################
# First, we can import useful libraries that we will use it later on

########################################################

import numpy as np
import scipy.sparse as sps
from scipy.special import expit
from copy import deepcopy
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler

########################################################
# Import ``GLM`` class from ``pyglmnet``

########################################################

# import GLM model
from pyglmnet import GLM

# create regularization parameters for model
reg_lambda = np.logspace(np.log(0.5), np.log(0.01), 10, base=np.exp(1))
glm_poissonexp = GLM(distr='poissonexp', verbose=False, alpha=0.05,
            max_iter=1000, learning_rate=1e-5,
            reg_lambda=reg_lambda, eta=4.0)

########################################################
#
#
# .. math::
#
#     J = \sum_i \lambda_i - y_i \log \lambda_i
#
# where
#
# .. math::
#
#     \lambda_i =
#     \begin{cases}
#         \exp(z_i), & \text{if}\ z_i \leq \eta \\
#         \\
#          \exp(\eta)z_i + (1-\eta)\exp(\eta), & \text{if}\ z_i \gt \eta
#     \end{cases}
#
# and
#
# .. math::
#
#     z_i = \beta_0 + \sum_j \beta_j x_{ij}
#
# Taking gradients,
#
# .. math::
#
#     \frac{\partial J}{\partial \beta_j} = \sum_i \frac{\partial J}{\partial \lambda_i} \frac{\partial \lambda_i}{\partial z_i} \frac{\partial z_i}{\partial \beta_j}
#
#
# .. math::
#
#     \frac{\partial J}{\partial \beta_0} =
#     \begin{cases}
#         \sum_i \Big(\lambda_i - y_i\Big), & \text{if}\ z_i \leq \eta \\
#         \\
#         \exp(\eta) \sum_i \Big(1 - \frac{\lambda_i}{y_i}\Big), & \text{if}\ z_i \gt \eta
#     \end{cases}
#
# .. math::
#     \frac{\partial J}{\partial \beta_j} =
#     \begin{cases}
#         \sum_i \Big(\lambda_i - y_i\Big)x_{ij}, & \text{if}\ z_i \leq \eta \\
#         \\
#         \exp(\eta) \sum_i \Big(1 - \frac{\lambda_i}{y_i}\Big)x_{ij}, & \text{if}\ z_i \gt \eta
#     \end{cases}
#
#
#

########################################################

########################################################

z = np.linspace(0., 10., 100)

eta = 4.0
qu = deepcopy(z)
slope = np.exp(eta)
intercept = (1 - eta) * slope
qu[z > eta] = z[z > eta] * slope + intercept
qu[z <= eta] = np.exp(z[z <= eta])

plt.plot(z, qu, label='a')
plt.plot(z, np.exp(z), label='b')
plt.ylim([0, 1000])
plt.xlabel('x')
plt.ylabel('y')
plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=1,
           ncol=2, borderaxespad=0.)
plt.show()

########################################################

########################################################

# Dataset size
n_samples, n_features = 10000, 100

# baseline term
beta0 = np.random.normal(0.0, 1.0, 1)
# sparse model terms
beta = sps.rand(n_features, 1, 0.1)
beta = np.array(beta.todense())

# training data
Xr = np.random.normal(0.0, 1.0, [n_samples, n_features])
yr = glm_poissonexp.simulate(beta0, beta, Xr)

# testing data
Xt = np.random.normal(0.0, 1.0, [n_samples, n_features])
yt = glm_poissonexp.simulate(beta0, beta, Xt)

########################################################
# Fit model to training data

########################################################

scaler = StandardScaler().fit(Xr)
glm_poissonexp.fit(scaler.transform(Xr),yr);

########################################################
# Gradient of loss function

########################################################

grad_beta0, grad_beta = glm_poissonexp.grad_L2loss(glm_poissonexp.fit_[-1]['beta0'], glm_poissonexp.fit_[-1]['beta'], 0.01, Xr, yr)
print(grad_beta[:5])

########################################################
# Use one model to predict

########################################################

m = glm_poissonexp[-1]
this_model_param = m.fit_
yrhat = m.predict(scaler.transform(Xr))
ythat = m.predict(scaler.transform(Xt))

########################################################
# Visualize predicted output

########################################################

plt.plot(yt[:100], label='tr')
plt.plot(ythat[:100], 'r', label='pr')
plt.xlabel('samples')
plt.ylabel('true and predicted outputs')
plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=1,
           ncol=2, borderaxespad=0.)
plt.show()

########################################################
# compute pseudo R-square

########################################################

print(m.score(yt, ythat, np.mean(yr), method='pseudo_R2'))
<commit_msg>modified example to reflect _grad_L2loss pseudo-private status<commit_after># -*- coding: utf-8 -*-
"""
=============================================
Poisson Exponential Distribution
=============================================

This is an example demonstrating how pyglmnet with
poisson exponential distribution works.

"""

########################################################
# First, we can import useful libraries that we will use it later on

########################################################

import numpy as np
import scipy.sparse as sps
from scipy.special import expit
from copy import deepcopy
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler

########################################################
# Import ``GLM`` class from ``pyglmnet``

########################################################

# import GLM model
from pyglmnet import GLM

# create regularization parameters for model
reg_lambda = np.logspace(np.log(0.5), np.log(0.01), 10, base=np.exp(1))
glm_poissonexp = GLM(distr='poissonexp', verbose=False, alpha=0.05,
            max_iter=1000, learning_rate=1e-5,
            reg_lambda=reg_lambda, eta=4.0)

########################################################
#
#
# .. math::
#
#     J = \sum_i \lambda_i - y_i \log \lambda_i
#
# where
#
# .. math::
#
#     \lambda_i =
#     \begin{cases}
#         \exp(z_i), & \text{if}\ z_i \leq \eta \\
#         \\
#          \exp(\eta)z_i + (1-\eta)\exp(\eta), & \text{if}\ z_i \gt \eta
#     \end{cases}
#
# and
#
# .. math::
#
#     z_i = \beta_0 + \sum_j \beta_j x_{ij}
#
# Taking gradients,
#
# .. math::
#
#     \frac{\partial J}{\partial \beta_j} = \sum_i \frac{\partial J}{\partial \lambda_i} \frac{\partial \lambda_i}{\partial z_i} \frac{\partial z_i}{\partial \beta_j}
#
#
# .. math::
#
#     \frac{\partial J}{\partial \beta_0} =
#     \begin{cases}
#         \sum_i \Big(\lambda_i - y_i\Big), & \text{if}\ z_i \leq \eta \\
#         \\
#         \exp(\eta) \sum_i \Big(1 - \frac{\lambda_i}{y_i}\Big), & \text{if}\ z_i \gt \eta
#     \end{cases}
#
# .. math::
#     \frac{\partial J}{\partial \beta_j} =
#     \begin{cases}
#         \sum_i \Big(\lambda_i - y_i\Big)x_{ij}, & \text{if}\ z_i \leq \eta \\
#         \\
#         \exp(\eta) \sum_i \Big(1 - \frac{\lambda_i}{y_i}\Big)x_{ij}, & \text{if}\ z_i \gt \eta
#     \end{cases}
#
#
#

########################################################

########################################################

z = np.linspace(0., 10., 100)

eta = 4.0
qu = deepcopy(z)
slope = np.exp(eta)
intercept = (1 - eta) * slope
qu[z > eta] = z[z > eta] * slope + intercept
qu[z <= eta] = np.exp(z[z <= eta])

plt.plot(z, qu, label='a')
plt.plot(z, np.exp(z), label='b')
plt.ylim([0, 1000])
plt.xlabel('x')
plt.ylabel('y')
plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=1,
           ncol=2, borderaxespad=0.)
plt.show()

########################################################

########################################################

# Dataset size
n_samples, n_features = 10000, 100

# baseline term
beta0 = np.random.normal(0.0, 1.0, 1)
# sparse model terms
beta = sps.rand(n_features, 1, 0.1)
beta = np.array(beta.todense())

# training data
Xr = np.random.normal(0.0, 1.0, [n_samples, n_features])
yr = glm_poissonexp.simulate(beta0, beta, Xr)

# testing data
Xt = np.random.normal(0.0, 1.0, [n_samples, n_features])
yt = glm_poissonexp.simulate(beta0, beta, Xt)

########################################################
# Fit model to training data

########################################################

scaler = StandardScaler().fit(Xr)
glm_poissonexp.fit(scaler.transform(Xr),yr);

########################################################
# Gradient of loss function

########################################################

grad_beta0, grad_beta = glm_poissonexp._grad_L2loss(glm_poissonexp.fit_[-1]['beta0'], glm_poissonexp.fit_[-1]['beta'], 0.01, Xr, yr)
print(grad_beta[:5])

########################################################
# Use one model to predict

########################################################

m = glm_poissonexp[-1]
this_model_param = m.fit_
yrhat = m.predict(scaler.transform(Xr))
ythat = m.predict(scaler.transform(Xt))

########################################################
# Visualize predicted output

########################################################

plt.plot(yt[:100], label='tr')
plt.plot(ythat[:100], 'r', label='pr')
plt.xlabel('samples')
plt.ylabel('true and predicted outputs')
plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=1,
           ncol=2, borderaxespad=0.)
plt.show()

########################################################
# compute pseudo R-square

########################################################

print(m.score(yt, ythat, np.mean(yr), method='pseudo_R2'))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2016 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

import keras.backend as K
from keras.models import Sequential
from keras.models import Model
from keras.layers import Layer
from keras.layers import LSTM
from keras.layers import Dense
from keras.layers import Input
from keras.layers import merge

class L2Normalize(Layer):
    def __init__(self, axis=-1, **kwargs):
        self.axis = axis
        return super(L2Normalize, self).__init__(**kwargs)

    def call(self, x, mask=None):
        return K.l2_normalize(x, axis=self.axis)

    def get_config(self):
        base_config = super(L2Normalize, self).get_config()
        base_config.update(dict(axis=self.axis))
        return base_config

class TripletLossSequenceEmbedding(object):
    """

    Parameters
    ----------
    input_shape: (n_samples, n_features) tuple
        Shape of input sequences.
    output_dim: int
        Embedding dimension.
    lstm: list
        List of output dimension of stacked LSTMs.
        Defaults to [12, ] (i.e. one LSTM with output dimension 12)
    dense: list
        List of output dimension ofr stacked dense layers.
        Defaults to [12, ] (i.e. one dense layer with output dimension 12)

    """
    def __init__(self, input_shape, output_dim, lstm=[12], dense=[12]):
        super(TripletLossSequenceEmbedding, self).__init__()
        self.input_shape = input_shape
        self.output_dim = output_dim
        self.lstm = lstm
        self.dense = dense

    def _embedding(self, shape):

        model = Sequential(name="embedding")

        # add LSTM layers
        n_lstm = len(self.lstm)
        for i, output_dim in enumerate(self.lstm):
            input_shape = self.input_shape if i == 0 else None
            return_sequences = i+1 < n_lstm
            layer = LSTM(input_shape=input_shape,
                         output_dim=output_dim,
                         return_sequences=return_sequences,
                         activation='tanh')
            model.add(layer)

        # add Dense layers
        for i, output_dim in enumerate(self.dense):
            layer = Dense(output_dim, activation='tanh')
            model.add(layer)

        #
        layer = Dense(self.output_dim, activation='tanh')
        model.add(layer)

        # add L2 normalization layer
        model.add(L2Normalize())

        return model

    @staticmethod
    def _triplet_loss(inputs, alpha=0.2):
        p = K.sum(K.square(inputs[0] - inputs[1]), axis=-1)
        n = K.sum(K.square(inputs[0] - inputs[2]), axis=-1)
        return K.maximum(0, p + alpha - n)

    @staticmethod
    def _output_shape(input_shapes):
        return (input_shapes[0][0], 1)

    @staticmethod
    def _identity_loss(y_true, y_pred):
        return K.mean(y_pred - 0 * y_true)

    def get_model(self, shape):

        anchor = Input(shape=shape, name="anchor")
        positive = Input(shape=shape, name="positive")
        negative = Input(shape=shape, name="negative")

        embed = self._embedding(shape)
        embedded_anchor = embed(anchor)
        embedded_positive = embed(positive)
        embedded_negative = embed(negative)

        distance = merge(
            [embedded_anchor, embedded_positive, embedded_negative],
            mode=self._triplet_loss, output_shape=self._output_shape)

        model = Model(input=[anchor, positive, negative], output=distance)

        model.compile(optimizer='rmsprop', loss=self._identity_loss)

        return model

    def get_embedding(self, from_model):
        return from_model.get_layer(name="embedding")
<commit_msg>chore: make embedding agnostic of sequence shape<commit_after>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2016 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

import keras.backend as K
from keras.models import Sequential
from keras.models import Model
from keras.layers import Layer
from keras.layers import LSTM
from keras.layers import Dense
from keras.layers import Input
from keras.layers import merge

class L2Normalize(Layer):
    def __init__(self, axis=-1, **kwargs):
        self.axis = axis
        return super(L2Normalize, self).__init__(**kwargs)

    def call(self, x, mask=None):
        return K.l2_normalize(x, axis=self.axis)

    def get_config(self):
        base_config = super(L2Normalize, self).get_config()
        base_config.update(dict(axis=self.axis))
        return base_config

class TripletLossSequenceEmbedding(object):
    """

    Parameters
    ----------
    output_dim: int
        Embedding dimension.
    lstm: list
        List of output dimension of stacked LSTMs.
        Defaults to [12, ] (i.e. one LSTM with output dimension 12)
    dense: list
        List of output dimension of additionnal stacked dense layers.
        Defaults to [] (i.e. do not add any dense layer)

    """
    def __init__(self, output_dim, lstm=[12], dense=[]):
        super(TripletLossSequenceEmbedding, self).__init__()
        self.output_dim = output_dim
        self.lstm = lstm
        self.dense = dense

    def _embedding(self, input_shape):

        model = Sequential(name="embedding")

        # stack LSTM layers
        n_lstm = len(self.lstm)
        for i, output_dim in enumerate(self.lstm):
            return_sequences = i+1 < n_lstm
            layer = LSTM(input_shape=input_shape if i==0 else None,
                         output_dim=output_dim,
                         return_sequences=return_sequences,
                         activation='tanh')
            model.add(layer)

        # stack dense layers
        for i, output_dim in enumerate(self.dense):
            layer = Dense(output_dim, activation='tanh')
            model.add(layer)

        # stack final dense layer
        layer = Dense(self.output_dim, activation='tanh')
        model.add(layer)

        # stack L2 normalization layer
        model.add(L2Normalize())

        return model

    @staticmethod
    def _triplet_loss(inputs, alpha=0.2):
        p = K.sum(K.square(inputs[0] - inputs[1]), axis=-1)
        n = K.sum(K.square(inputs[0] - inputs[2]), axis=-1)
        return K.maximum(0, p + alpha - n)

    @staticmethod
    def _output_shape(input_shapes):
        return (input_shapes[0][0], 1)

    @staticmethod
    def _identity_loss(y_true, y_pred):
        return K.mean(y_pred - 0 * y_true)

    def get_model(self, input_shape):
        """
        Parameters
        ----------
        input_shape: (n_samples, n_features) tuple
            Shape of input sequences.
        """

        anchor = Input(shape=input_shape, name="anchor")
        positive = Input(shape=input_shape, name="positive")
        negative = Input(shape=input_shape, name="negative")

        embed = self._embedding(input_shape)
        embedded_anchor = embed(anchor)
        embedded_positive = embed(positive)
        embedded_negative = embed(negative)

        distance = merge(
            [embedded_anchor, embedded_positive, embedded_negative],
            mode=self._triplet_loss, output_shape=self._output_shape)

        model = Model(input=[anchor, positive, negative], output=distance)

        model.compile(optimizer='rmsprop', loss=self._identity_loss)

        return model

    def get_embedding(self, from_model):
        return from_model.get_layer(name="embedding")
<|endoftext|>


<commit_before>#
# This file is part of pyasn1 software.
#
# Copyright (c) 2005-2017, Ilya Etingof <etingof@gmail.com>
# License: http://pyasn1.sf.net/license.html
#
import sys
from pyasn1.type import tag, tagmap
from pyasn1 import error

__all__ = ['NamedType', 'OptionalNamedType', 'DefaultedNamedType', 'NamedTypes']

class NamedType(object):
    """Create named field object for a constructed ASN.1 type.

    The |NamedType| object represents a single name and ASN.1 type of a constructed ASN.1 type.

    |NamedType| objects are immutable and duck-type Python :class:`tuple` objects
    holding *name* and *asn1Object* components.

    Parameters
    ----------
    name: :py:class:`str`
        Field name

    asn1Object:
        ASN.1 type object
    """
    isOptional = False
    isDefaulted = False

    def __init__(self, name, asn1Object):
        self.__name = name
        self.__type = asn1Object
        self.__nameAndType = name, asn1Object

    def __repr__(self):
        return '%s(%r, %r)' % (self.__class__.__name__, self.__name, self.__type)

    def __eq__(self, other):
        return self.__nameAndType == other

    def __ne__(self, other):
        return self.__nameAndType != other

    def __lt__(self, other):
        return self.__nameAndType < other

    def __le__(self, other):
        return self.__nameAndType <= other

    def __gt__(self, other):
        return self.__nameAndType > other

    def __ge__(self, other):
        return self.__nameAndType >= other

    def __hash__(self):
        return hash(self.__nameAndType)

    def __getitem__(self, idx):
        return self.__nameAndType[idx]

    def __iter__(self):
        return iter(self.__nameAndType)

    @property
    def name(self):
        return self.__name

    @property
    def asn1Object(self):
        return self.__type

    # Backward compatibility

    def getName(self):
        return self.name

    def getType(self):
        return self.asn1Object

class OptionalNamedType(NamedType):
    __doc__ = NamedType.__doc__

    isOptional = True

class DefaultedNamedType(NamedType):
    __doc__ = NamedType.__doc__

    isDefaulted = True

class NamedTypes(object):
    """Create a collection of named fields for a constructed ASN.1 type.

    The NamedTypes object represents a collection of named fields of a constructed ASN.1 type.

    *NamedTypes* objects are immutable and duck-type Python :class:`dict` objects
    holding *name* as keys and ASN.1 type object as values.

    Parameters
    ----------
    *namedTypes: :class:`~pyasn1.type.namedtype.NamedType`
    """
    def __init__(self, *namedTypes):
        self.__namedTypes = namedTypes
        self.__namedTypesLen = len(self.__namedTypes)
        self.__minTagSet = self.__computeMinTagSet()
        self.__nameToPosMap = self.__computeNameToPosMap()
        self.__tagToPosMap = self.__computeTagToPosMap()
        self.__ambiguousTypes = self.__computeAmbiguousTypes()
        self.__uniqueTagMap = self.__computeTagMaps(unique=True)
        self.__nonUniqueTagMap = self.__computeTagMaps(unique=False)
        self.__hasOptionalOrDefault = bool([True for namedType in self.__namedTypes
                                            if namedType.isDefaulted or namedType.isOptional])
        self.__requiredComponents = frozenset(
                [idx for idx, nt in enumerate(self.__namedTypes) if not nt.isOptional and not nt.isDefaulted]
            )

    def __repr__(self):
        return '%s(%s)' % (
            self.__class__.__name__, ', '.join([repr(x) for x in self.__namedTypes])
        )

    def __eq__(self, other):
        return self.__namedTypes == other

    def __ne__(self, other):
        return self.__namedTypes != other

    def __lt__(self, other):
        return self.__namedTypes < other

    def __le__(self, other):
        return self.__namedTypes <= other

    def __gt__(self, other):
        return self.__namedTypes > other

    def __ge__(self, other):
        return self.__namedTypes >= other

    def __hash__(self):
        return hash(self.__namedTypes)

    def __getitem__(self, idx):
        try:
            return self.__namedTypes[idx]

        except TypeError:
            return self.__namedTypes[self.__nameToPosMap[idx]]

    def __contains__(self, key):
        return key in self.__nameToPosMap

    def __iter__(self):
        return (x[0] for x in self.__namedTypes)

    if sys.version_info[0] <= 2:
        def __nonzero__(self):
            return self.__namedTypesLen > 0
    else:
        def __bool__(self):
            return self.__namedTypesLen > 0

    def __len__(self):
        return self.__namedTypesLen

    # Python dict protocol

    def values(self):
        return (namedType.asn1Object for namedType in self.__namedTypes)

    def keys(self):
        return (namedType.name for namedType in self.__namedTypes)

    def items(self):
        return ((namedType.name, namedType.asn1Object) for namedType in self.__namedTypes)

    def clone(self):
        return self.__class__(*self.__namedTypes)

    class PostponedError(object):
        def __init__(self, errorMsg):
            self.__errorMsg = errorMsg

        def __getitem__(self, item):
            raise  error.PyAsn1Error(self.__errorMsg)

    def __computeTagToPosMap(self):
        tagToPosMap = {}
        for idx, namedType in enumerate(self.__namedTypes):
            tagMap = namedType.asn1Object.tagMap
            if not tagMap:
                continue
            for _tagSet in tagMap.presentTypes:
                if _tagSet in tagToPosMap:
                    return NamedTypes.PostponedError('Duplicate component tag %s at %s' % (_tagSet, namedType))
                tagToPosMap[_tagSet] = idx

        return tagToPosMap

    def __computeNameToPosMap(self):
        nameToPosMap = {}
        for idx, namedType in enumerate(self.__namedTypes):
            if namedType.name in nameToPosMap:
                return NamedTypes.PostponedError('Duplicate component name %s at %s' % (namedType.name, namedType))
            nameToPosMap[namedType.name] = idx

        return nameToPosMap

    def __computeAmbiguousTypes(self):
        ambigiousTypes = {}
        partialAmbigiousTypes = ()
        for idx, namedType in reversed(tuple(enumerate(self.__namedTypes))):
            if namedType.isOptional or namedType.isDefaulted:
                partialAmbigiousTypes = (namedType,) + partialAmbigiousTypes
            else:
                partialAmbigiousTypes = (namedType,)
            if len(partialAmbigiousTypes) == len(self.__namedTypes):
                ambigiousTypes[idx] = self
            else:
                ambigiousTypes[idx] = NamedTypes(*partialAmbigiousTypes)
        return ambigiousTypes

    def getTypeByPosition(self, idx):
        """Return ASN.1 type object by its position in fields set.

        Parameters
        ----------
        idx: :py:class:`int`
            Field index

        Returns
        -------
        :
            ASN.1 type

        Raises
        ------
        : :class:`~pyasn1.error.PyAsn1Error`
            If given position is out of fields range
        """
        try:
            return self.__namedTypes[idx].asn1Object

        except IndexError:
            raise error.PyAsn1Error('Type position out of range')

    def getPositionByType(self, tagSet):
        """Return field position by its ASN.1 type.

        Parameters
        ----------
        tagSet: :class:`~pysnmp.type.tag.TagSet`
            ASN.1 tag set distinguishing one ASN.1 type from others.

        Returns
        -------
        : :py:class:`int`
            ASN.1 type position in fields set

        Raises
        ------
        : :class:`~pyasn1.error.PyAsn1Error`
            If *tagSet* is not present or ASN.1 types are not unique within callee *NamedTypes*
        """
        try:
            return self.__tagToPosMap[tagSet]

        except KeyError:
            raise error.PyAsn1Error('Type %s not found' % (tagSet,))

    def getNameByPosition(self, idx):
        """Return field name by its position in fields set.

        Parameters
        ----------
        idx: :py:class:`idx`
            Field index

        Returns
        -------
        : :py:class:`str`
            Field name

        Raises
        ------
        : :class:`~pyasn1.error.PyAsn1Error`
            If given field name is not present in callee *NamedTypes*
        """
        try:
            return self.__namedTypes[idx].name

        except IndexError:
            raise error.PyAsn1Error('Type position out of range')

    def getPositionByName(self, name):
        """Return field position by filed name.

        Parameters
        ----------
        name: :py:class:`str`
            Field name

        Returns
        -------
        : :py:class:`int`
            Field position in fields set

        Raises
        ------
        : :class:`~pyasn1.error.PyAsn1Error`
            If *name* is not present or not unique within callee *NamedTypes*
        """
        try:
            return self.__nameToPosMap[name]

        except KeyError:
            raise error.PyAsn1Error('Name %s not found' % (name,))

    def getTagMapNearPosition(self, idx):
        """Return ASN.1 types that are allowed at or past given field position.

        Some ASN.1 serialization allow for skipping optional and defaulted fields.
        Some constructed ASN.1 types allow reordering of the fields. When recovering
        such objects it may be important to know which types can possibly be
        present at any given position in the field sets.

        Parameters
        ----------
        idx: :py:class:`int`
            Field index

        Returns
        -------
        : :class:`~pyasn1.type.tagmap.TagMap`
            Map if ASN.1 types allowed at given field position

        Raises
        ------
        : :class:`~pyasn1.error.PyAsn1Error`
            If given position is out of fields range
        """
        try:
            return self.__ambiguousTypes[idx].tagMap

        except KeyError:
            raise error.PyAsn1Error('Type position out of range')

    def getPositionNearType(self, tagSet, idx):
        """Return the closest field position where given ASN.1 type is allowed.

        Some ASN.1 serialization allow for skipping optional and defaulted fields.
        Some constructed ASN.1 types allow reordering of the fields. When recovering
        such objects it may be important to know at which field position, in field set,
        given *tagSet* is allowed at or past *idx* position.

        Parameters
        ----------
        tagSet: :class:`~pyasn1.type.tag.TagSet`
           ASN.1 type which field position to look up

        idx: :py:class:`int`
            Field position at or past which to perform ASN.1 type look up

        Returns
        -------
        : :py:class:`int`
            Field position in fields set

        Raises
        ------
        : :class:`~pyasn1.error.PyAsn1Error`
            If *tagSet* is not present or not unique within callee *NamedTypes*
            or *idx* is out of fields range
        """
        try:
            return idx + self.__ambiguousTypes[idx].getPositionByType(tagSet)

        except KeyError:
            raise error.PyAsn1Error('Type position out of range')

    def __computeMinTagSet(self):
        minTagSet = None
        for namedType in self.__namedTypes:
            asn1Object = namedType.asn1Object
            try:
                tagSet = asn1Object.minTagSet

            except AttributeError:
                tagSet = asn1Object.tagSet
            if minTagSet is None or tagSet < minTagSet:
                minTagSet = tagSet
        return minTagSet or tag.TagSet()

    @property
    def minTagSet(self):
        """Return the minimal TagSet among ASN.1 type in callee *NamedTypes*.

        Some ASN.1 types/serialization protocols require ASN.1 types to be
        arranged based on their numerical tag value. The *minTagSet* property
        returns that.

        Returns
        -------
        : :class:`~pyasn1.type.tagset.TagSet`
            Minimal TagSet among ASN.1 types in callee *NamedTypes*
        """
        return self.__minTagSet

    def __computeTagMaps(self, unique):
        presentTypes = {}
        skipTypes = {}
        defaultType = None
        for namedType in self.__namedTypes:
            tagMap = namedType.asn1Object.tagMap
            for tagSet in tagMap:
                if unique and tagSet in presentTypes:
                    return NamedTypes.PostponedError('Non-unique tagSet %s of %s at %s' % (tagSet, namedType, self))
                presentTypes[tagSet] = namedType.asn1Object
            skipTypes.update(tagMap.skipTypes)

            if defaultType is None:
                defaultType = tagMap.defaultType
            elif tagMap.defaultType is not None:
                raise error.PyAsn1Error('Duplicate default ASN.1 type at %s' % (self,))

        return tagmap.TagMap(presentTypes, skipTypes, defaultType)

    @property
    def tagMap(self):
        """Return a *TagMap* object from tags and types recursively.

        Return a :class:`~pyasn1.type.tagmap.TagMap` object by
        combining tags from *TagMap* objects of children types and
        associating them with their immediate child type.

        Example
        -------

        .. code-block:: python

           OuterType ::= CHOICE {
               innerType INTEGER
           }

        Calling *.tagMap* on *OuterType* will yield a map like this:

        .. code-block:: python

           Integer.tagSet -> Choice
        """
        return self.__nonUniqueTagMap

    @property
    def tagMapUnique(self):
        """Return a *TagMap* object from unique tags and types recursively.

        Return a :class:`~pyasn1.type.tagmap.TagMap` object by
        combining tags from *TagMap* objects of children types and
        associating them with their immediate child type.

        Example
        -------

        .. code-block:: python

           OuterType ::= CHOICE {
               innerType INTEGER
           }

        Calling *.tagMapUnique* on *OuterType* will yield a map like this:

        .. code-block:: python

           Integer.tagSet -> Choice

        Note
        ----

        Duplicate *TagSet* objects found in the tree of children
        types would cause error.
        """
        return self.__uniqueTagMap

    @property
    def hasOptionalOrDefault(self):
        return self.__hasOptionalOrDefault

    @property
    def namedTypes(self):
        return iter(self.__namedTypes)

    @property
    def requiredComponents(self):
        return self.__requiredComponents
<commit_msg>account for postponed errors when computing type maps<commit_after>#
# This file is part of pyasn1 software.
#
# Copyright (c) 2005-2017, Ilya Etingof <etingof@gmail.com>
# License: http://pyasn1.sf.net/license.html
#
import sys
from pyasn1.type import tag, tagmap
from pyasn1 import error

__all__ = ['NamedType', 'OptionalNamedType', 'DefaultedNamedType', 'NamedTypes']

class NamedType(object):
    """Create named field object for a constructed ASN.1 type.

    The |NamedType| object represents a single name and ASN.1 type of a constructed ASN.1 type.

    |NamedType| objects are immutable and duck-type Python :class:`tuple` objects
    holding *name* and *asn1Object* components.

    Parameters
    ----------
    name: :py:class:`str`
        Field name

    asn1Object:
        ASN.1 type object
    """
    isOptional = False
    isDefaulted = False

    def __init__(self, name, asn1Object):
        self.__name = name
        self.__type = asn1Object
        self.__nameAndType = name, asn1Object

    def __repr__(self):
        return '%s(%r, %r)' % (self.__class__.__name__, self.__name, self.__type)

    def __eq__(self, other):
        return self.__nameAndType == other

    def __ne__(self, other):
        return self.__nameAndType != other

    def __lt__(self, other):
        return self.__nameAndType < other

    def __le__(self, other):
        return self.__nameAndType <= other

    def __gt__(self, other):
        return self.__nameAndType > other

    def __ge__(self, other):
        return self.__nameAndType >= other

    def __hash__(self):
        return hash(self.__nameAndType)

    def __getitem__(self, idx):
        return self.__nameAndType[idx]

    def __iter__(self):
        return iter(self.__nameAndType)

    @property
    def name(self):
        return self.__name

    @property
    def asn1Object(self):
        return self.__type

    # Backward compatibility

    def getName(self):
        return self.name

    def getType(self):
        return self.asn1Object

class OptionalNamedType(NamedType):
    __doc__ = NamedType.__doc__

    isOptional = True

class DefaultedNamedType(NamedType):
    __doc__ = NamedType.__doc__

    isDefaulted = True

class NamedTypes(object):
    """Create a collection of named fields for a constructed ASN.1 type.

    The NamedTypes object represents a collection of named fields of a constructed ASN.1 type.

    *NamedTypes* objects are immutable and duck-type Python :class:`dict` objects
    holding *name* as keys and ASN.1 type object as values.

    Parameters
    ----------
    *namedTypes: :class:`~pyasn1.type.namedtype.NamedType`
    """
    def __init__(self, *namedTypes):
        self.__namedTypes = namedTypes
        self.__namedTypesLen = len(self.__namedTypes)
        self.__minTagSet = self.__computeMinTagSet()
        self.__nameToPosMap = self.__computeNameToPosMap()
        self.__tagToPosMap = self.__computeTagToPosMap()
        self.__ambiguousTypes = self.__computeAmbiguousTypes()
        self.__uniqueTagMap = self.__computeTagMaps(unique=True)
        self.__nonUniqueTagMap = self.__computeTagMaps(unique=False)
        self.__hasOptionalOrDefault = bool([True for namedType in self.__namedTypes
                                            if namedType.isDefaulted or namedType.isOptional])
        self.__requiredComponents = frozenset(
                [idx for idx, nt in enumerate(self.__namedTypes) if not nt.isOptional and not nt.isDefaulted]
            )

    def __repr__(self):
        return '%s(%s)' % (
            self.__class__.__name__, ', '.join([repr(x) for x in self.__namedTypes])
        )

    def __eq__(self, other):
        return self.__namedTypes == other

    def __ne__(self, other):
        return self.__namedTypes != other

    def __lt__(self, other):
        return self.__namedTypes < other

    def __le__(self, other):
        return self.__namedTypes <= other

    def __gt__(self, other):
        return self.__namedTypes > other

    def __ge__(self, other):
        return self.__namedTypes >= other

    def __hash__(self):
        return hash(self.__namedTypes)

    def __getitem__(self, idx):
        try:
            return self.__namedTypes[idx]

        except TypeError:
            return self.__namedTypes[self.__nameToPosMap[idx]]

    def __contains__(self, key):
        return key in self.__nameToPosMap

    def __iter__(self):
        return (x[0] for x in self.__namedTypes)

    if sys.version_info[0] <= 2:
        def __nonzero__(self):
            return self.__namedTypesLen > 0
    else:
        def __bool__(self):
            return self.__namedTypesLen > 0

    def __len__(self):
        return self.__namedTypesLen

    # Python dict protocol

    def values(self):
        return (namedType.asn1Object for namedType in self.__namedTypes)

    def keys(self):
        return (namedType.name for namedType in self.__namedTypes)

    def items(self):
        return ((namedType.name, namedType.asn1Object) for namedType in self.__namedTypes)

    def clone(self):
        return self.__class__(*self.__namedTypes)

    class PostponedError(object):
        def __init__(self, errorMsg):
            self.__errorMsg = errorMsg

        def __getitem__(self, item):
            raise  error.PyAsn1Error(self.__errorMsg)

    def __computeTagToPosMap(self):
        tagToPosMap = {}
        for idx, namedType in enumerate(self.__namedTypes):
            tagMap = namedType.asn1Object.tagMap
            if isinstance(tagMap, NamedTypes.PostponedError):
                return tagMap
            if not tagMap:
                continue
            for _tagSet in tagMap.presentTypes:
                if _tagSet in tagToPosMap:
                    return NamedTypes.PostponedError('Duplicate component tag %s at %s' % (_tagSet, namedType))
                tagToPosMap[_tagSet] = idx

        return tagToPosMap

    def __computeNameToPosMap(self):
        nameToPosMap = {}
        for idx, namedType in enumerate(self.__namedTypes):
            if namedType.name in nameToPosMap:
                return NamedTypes.PostponedError('Duplicate component name %s at %s' % (namedType.name, namedType))
            nameToPosMap[namedType.name] = idx

        return nameToPosMap

    def __computeAmbiguousTypes(self):
        ambigiousTypes = {}
        partialAmbigiousTypes = ()
        for idx, namedType in reversed(tuple(enumerate(self.__namedTypes))):
            if namedType.isOptional or namedType.isDefaulted:
                partialAmbigiousTypes = (namedType,) + partialAmbigiousTypes
            else:
                partialAmbigiousTypes = (namedType,)
            if len(partialAmbigiousTypes) == len(self.__namedTypes):
                ambigiousTypes[idx] = self
            else:
                ambigiousTypes[idx] = NamedTypes(*partialAmbigiousTypes)
        return ambigiousTypes

    def getTypeByPosition(self, idx):
        """Return ASN.1 type object by its position in fields set.

        Parameters
        ----------
        idx: :py:class:`int`
            Field index

        Returns
        -------
        :
            ASN.1 type

        Raises
        ------
        : :class:`~pyasn1.error.PyAsn1Error`
            If given position is out of fields range
        """
        try:
            return self.__namedTypes[idx].asn1Object

        except IndexError:
            raise error.PyAsn1Error('Type position out of range')

    def getPositionByType(self, tagSet):
        """Return field position by its ASN.1 type.

        Parameters
        ----------
        tagSet: :class:`~pysnmp.type.tag.TagSet`
            ASN.1 tag set distinguishing one ASN.1 type from others.

        Returns
        -------
        : :py:class:`int`
            ASN.1 type position in fields set

        Raises
        ------
        : :class:`~pyasn1.error.PyAsn1Error`
            If *tagSet* is not present or ASN.1 types are not unique within callee *NamedTypes*
        """
        try:
            return self.__tagToPosMap[tagSet]

        except KeyError:
            raise error.PyAsn1Error('Type %s not found' % (tagSet,))

    def getNameByPosition(self, idx):
        """Return field name by its position in fields set.

        Parameters
        ----------
        idx: :py:class:`idx`
            Field index

        Returns
        -------
        : :py:class:`str`
            Field name

        Raises
        ------
        : :class:`~pyasn1.error.PyAsn1Error`
            If given field name is not present in callee *NamedTypes*
        """
        try:
            return self.__namedTypes[idx].name

        except IndexError:
            raise error.PyAsn1Error('Type position out of range')

    def getPositionByName(self, name):
        """Return field position by filed name.

        Parameters
        ----------
        name: :py:class:`str`
            Field name

        Returns
        -------
        : :py:class:`int`
            Field position in fields set

        Raises
        ------
        : :class:`~pyasn1.error.PyAsn1Error`
            If *name* is not present or not unique within callee *NamedTypes*
        """
        try:
            return self.__nameToPosMap[name]

        except KeyError:
            raise error.PyAsn1Error('Name %s not found' % (name,))

    def getTagMapNearPosition(self, idx):
        """Return ASN.1 types that are allowed at or past given field position.

        Some ASN.1 serialization allow for skipping optional and defaulted fields.
        Some constructed ASN.1 types allow reordering of the fields. When recovering
        such objects it may be important to know which types can possibly be
        present at any given position in the field sets.

        Parameters
        ----------
        idx: :py:class:`int`
            Field index

        Returns
        -------
        : :class:`~pyasn1.type.tagmap.TagMap`
            Map if ASN.1 types allowed at given field position

        Raises
        ------
        : :class:`~pyasn1.error.PyAsn1Error`
            If given position is out of fields range
        """
        try:
            return self.__ambiguousTypes[idx].tagMap

        except KeyError:
            raise error.PyAsn1Error('Type position out of range')

    def getPositionNearType(self, tagSet, idx):
        """Return the closest field position where given ASN.1 type is allowed.

        Some ASN.1 serialization allow for skipping optional and defaulted fields.
        Some constructed ASN.1 types allow reordering of the fields. When recovering
        such objects it may be important to know at which field position, in field set,
        given *tagSet* is allowed at or past *idx* position.

        Parameters
        ----------
        tagSet: :class:`~pyasn1.type.tag.TagSet`
           ASN.1 type which field position to look up

        idx: :py:class:`int`
            Field position at or past which to perform ASN.1 type look up

        Returns
        -------
        : :py:class:`int`
            Field position in fields set

        Raises
        ------
        : :class:`~pyasn1.error.PyAsn1Error`
            If *tagSet* is not present or not unique within callee *NamedTypes*
            or *idx* is out of fields range
        """
        try:
            return idx + self.__ambiguousTypes[idx].getPositionByType(tagSet)

        except KeyError:
            raise error.PyAsn1Error('Type position out of range')

    def __computeMinTagSet(self):
        minTagSet = None
        for namedType in self.__namedTypes:
            asn1Object = namedType.asn1Object
            try:
                tagSet = asn1Object.minTagSet

            except AttributeError:
                tagSet = asn1Object.tagSet
            if minTagSet is None or tagSet < minTagSet:
                minTagSet = tagSet
        return minTagSet or tag.TagSet()

    @property
    def minTagSet(self):
        """Return the minimal TagSet among ASN.1 type in callee *NamedTypes*.

        Some ASN.1 types/serialization protocols require ASN.1 types to be
        arranged based on their numerical tag value. The *minTagSet* property
        returns that.

        Returns
        -------
        : :class:`~pyasn1.type.tagset.TagSet`
            Minimal TagSet among ASN.1 types in callee *NamedTypes*
        """
        return self.__minTagSet

    def __computeTagMaps(self, unique):
        presentTypes = {}
        skipTypes = {}
        defaultType = None
        for namedType in self.__namedTypes:
            tagMap = namedType.asn1Object.tagMap
            if isinstance(tagMap, NamedTypes.PostponedError):
                return tagMap
            for tagSet in tagMap:
                if unique and tagSet in presentTypes:
                    return NamedTypes.PostponedError('Non-unique tagSet %s of %s at %s' % (tagSet, namedType, self))
                presentTypes[tagSet] = namedType.asn1Object
            skipTypes.update(tagMap.skipTypes)

            if defaultType is None:
                defaultType = tagMap.defaultType
            elif tagMap.defaultType is not None:
                raise error.PyAsn1Error('Duplicate default ASN.1 type at %s' % (self,))

        return tagmap.TagMap(presentTypes, skipTypes, defaultType)

    @property
    def tagMap(self):
        """Return a *TagMap* object from tags and types recursively.

        Return a :class:`~pyasn1.type.tagmap.TagMap` object by
        combining tags from *TagMap* objects of children types and
        associating them with their immediate child type.

        Example
        -------

        .. code-block:: python

           OuterType ::= CHOICE {
               innerType INTEGER
           }

        Calling *.tagMap* on *OuterType* will yield a map like this:

        .. code-block:: python

           Integer.tagSet -> Choice
        """
        return self.__nonUniqueTagMap

    @property
    def tagMapUnique(self):
        """Return a *TagMap* object from unique tags and types recursively.

        Return a :class:`~pyasn1.type.tagmap.TagMap` object by
        combining tags from *TagMap* objects of children types and
        associating them with their immediate child type.

        Example
        -------

        .. code-block:: python

           OuterType ::= CHOICE {
               innerType INTEGER
           }

        Calling *.tagMapUnique* on *OuterType* will yield a map like this:

        .. code-block:: python

           Integer.tagSet -> Choice

        Note
        ----

        Duplicate *TagSet* objects found in the tree of children
        types would cause error.
        """
        return self.__uniqueTagMap

    @property
    def hasOptionalOrDefault(self):
        return self.__hasOptionalOrDefault

    @property
    def namedTypes(self):
        return iter(self.__namedTypes)

    @property
    def requiredComponents(self):
        return self.__requiredComponents
<|endoftext|>


<commit_before># python_example.py

# Steve Ivy <steveivy@gmail.com>
# http://monkinetic.com

# Sends statistics to the stats daemon over UDP
class StatsdClient(object):
    def __init__(self, host='localhost', port=8125):
        self.addr=(host, port)

    def timing(self, stat, time, sample_rate=1):
        """
        Log timing information
        >>> from python_example import Statsd
        >>> Statsd.timing('some.time', 500)
        """
        stats = {}
        stats[stat] = "%d|ms" % time
        self.send(stats, sample_rate)

    def increment(self, stats, sample_rate=1):
        """
        Increments one or more stats counters
        >>> Statsd.increment('some.int')
        >>> Statsd.increment('some.int',0.5)
        """
        self.update_stats(stats, 1, sample_rate)

    def decrement(self, stats, sample_rate=1):
        """
        Decrements one or more stats counters
        >>> Statsd.decrement('some.int')
        """
        self.update_stats(stats, -1, sample_rate)

    def update_stats(self, stats, delta=1, sampleRate=1):
        """
        Updates one or more stats counters by arbitrary amounts
        >>> Statsd.update_stats('some.int',10)
        """
        if isinstance(stats, list):
            stats = [stats]
        data = {}
        for stat in stats:
            data[stat] = "%s|c" % delta

        self.send(data, sampleRate)

    def send(self, data, sample_rate=1):
        """
        Squirt the metrics over UDP
        """
        sampled_data = {}

        if(sample_rate < 1):
            import random
            if random.random() <= sample_rate:
                for stat, value in data.items():
                    sampled_data[stat] = "%s|@%s" %(value, sample_rate)
        else:
            sampled_data=data

        from socket import socket, AF_INET, SOCK_DGRAM
        udp_sock = socket(AF_INET, SOCK_DGRAM)
        try:
            for stat, value in sampled_data.items():
                send_data = "%s:%s" % (stat, value)
                udp_sock.sendto(send_data, self.addr)
        except Exception:
            import sys
            import traceback
            print >>sys.stderr, "Unexpected error: ", traceback.format_exc()
            return False
        return True

if __name__=="__main__":
    c = StatsdClient()
    c.increment('example.python')
<commit_msg>exmaples: python: unbreak doctests<commit_after># python_example.py

# Steve Ivy <steveivy@gmail.com>
# http://monkinetic.com

# Sends statistics to the stats daemon over UDP
class StatsdClient(object):
    def __init__(self, host='localhost', port=8125):
        self.addr=(host, port)

    def timing(self, stat, time, sample_rate=1):
        """
        Log timing information
        >>> from python_example import StatsdClient
        >>> client = StatsdClient()
        >>> client.timing('some.time', 500)
        """
        stats = {}
        stats[stat] = "%d|ms" % time
        self.send(stats, sample_rate)

    def increment(self, stats, sample_rate=1):
        """
        Increments one or more stats counters
        >>> client = StatsdClient()
        >>> client.increment('some.int')
        >>> client.increment('some.int',0.5)
        """
        self.update_stats(stats, 1, sample_rate)

    def decrement(self, stats, sample_rate=1):
        """
        Decrements one or more stats counters
        >>> client = StatsdClient()
        >>> client.decrement('some.int')
        """
        self.update_stats(stats, -1, sample_rate)

    def update_stats(self, stats, delta=1, sampleRate=1):
        """
        Updates one or more stats counters by arbitrary amounts
        >>> client = StatsdClient()
        >>> client.update_stats('some.int',10)
        """
        if isinstance(stats, list):
            stats = [stats]
        data = {}
        for stat in stats:
            data[stat] = "%s|c" % delta

        self.send(data, sampleRate)

    def send(self, data, sample_rate=1):
        """
        Squirt the metrics over UDP
        """
        sampled_data = {}

        if(sample_rate < 1):
            import random
            if random.random() <= sample_rate:
                for stat, value in data.items():
                    sampled_data[stat] = "%s|@%s" %(value, sample_rate)
        else:
            sampled_data=data

        from socket import socket, AF_INET, SOCK_DGRAM
        udp_sock = socket(AF_INET, SOCK_DGRAM)
        try:
            for stat, value in sampled_data.items():
                send_data = "%s:%s" % (stat, value)
                udp_sock.sendto(send_data, self.addr)
        except Exception:
            import sys
            import traceback
            print >>sys.stderr, "Unexpected error: ", traceback.format_exc()
            return False
        return True

if __name__=="__main__":
    c = StatsdClient()
    c.increment('example.python')
<|endoftext|>