<commit_before># -*- coding: utf-8 -*-

from operator import itemgetter

import matplotlib.pyplot as plt
from cmath import sqrt
from matplotlib import mlab

def truncate(x, d):
    return int(x * (10.0**d)) / (10.0**d)

class CorpusInformation:
    def __init__(self, word_lists, l_files):
        """
        takes in wordlists and convert that completely to statistic and give
        anomalies (about file size)

        :param word_lists: an array contain dictionaries map from word to word
                            count
                            each dictionary is word count of a particular file
        :param l_files: an parallel array of WordLists, contain file name of
                            the files(in order to plot)
        """

        # initialize
        num_file = len(word_lists)
        file_anomaly_std_err = {}
        file_anomaly_iqr = {}
        file_sizes = {}

        for i in range(num_file):
            file_sizes.update({l_files[i]: sum(word_lists[i].values())})
        # 1 standard error analysis
        average_file_size = sum(file_sizes.values()) / num_file
        # calculate the StdE
        std_err_file_size = 0
        for file_size in list(file_sizes.values()):
            std_err_file_size += (file_size - average_file_size) ** 2
        std_err_file_size /= num_file
        std_err_file_size = sqrt(std_err_file_size)
        # calculate the anomaly
        for file in l_files:
            if file_sizes[file] > average_file_size + 2 * std_err_file_size:
                file_anomaly_std_err.update({file.name: 'large'})
            elif file_sizes[file] < average_file_size - 2 * std_err_file_size:
                file_anomaly_std_err.update({file.name: 'small'})

        # 2 iqr analysis
        temp_list = sorted(list(file_sizes.items()), key=itemgetter(1))
        mid = temp_list[int(num_file / 2)][1]
        q3 = temp_list[int(num_file * 3 / 4)][1]
        q1 = temp_list[int(num_file / 4)][1]
        iqr = q3 - q1
        # calculate the anomaly
        for file in l_files:
            if file_sizes[file] > mid + 1.5 * iqr:
                file_anomaly_iqr.update({file.name: 'large'})
            elif file_sizes[file] < mid - 1.5 * iqr:
                file_anomaly_iqr.update({file.name: 'small'})

        # pack the data
        self.NumFile = num_file  # number of files
        # an array of the total word count of each file
        self.FileSizes = file_sizes
        self.Average = average_file_size  # average file size
        self.StdE = std_err_file_size  # standard error of file size
        # an array contain dictionary map anomaly file to
        # how they are different from others(too large or too small)
        # analyzed in using standard error
        self.FileAnomalyStdE = file_anomaly_std_err

        self.Q1 = q1  # q1 of a all the file sizes
        self.Median = mid  # median of all the file sizes
        self.Q3 = q3  # q1 of a all the file sizes
        self.IQR = iqr  # q1 of a all the file sizes
        # an array contain dictionary map anomaly file to
        # how they are different from others(too large or too small)
        # analyzed in using iqr
        self.FileAnomalyIQR = file_anomaly_iqr

    def list_stats(self):
        """
        print all the statistics in a good manner

        """
        print()
        print('average:', self.Average, ' standard error:', self.StdE)
        print(
            'document size anomaly calculated using standard error:',
            self.FileAnomalyStdE)
        print('median:', self.Median, ' Q1:', self.Q1, ' Q3:', self.Q3,
              ' IQR', self.IQR)
        print('document size anomaly calculated using IQR:',
              self.FileAnomalyIQR)

    def plot(self, path):
        """
        plot a bar chart to represent the statistics
        x is the file name
        y is the file size(using word count to represent)
        """
        plt.bar(
            list(
                range(
                    self.NumFile)), list(
                self.FileSizes.values()), align='center')
        plt.xticks(list(range(self.NumFile)), list(self.FileSizes.keys()))
        plt.xticks(rotation=50)
        plt.xlabel('File Name')
        plt.ylabel('File Size(in term of word count)')
        plt.savefig(path)
        plt.close()

    def return_statistics(self):
        """
        :return: a dictionary map the statistic name to the actual statistics
        """
        return {'average': truncate(self.Average, 3),
                'StdE': self.StdE,
                'fileanomalyStdE': self.FileAnomalyStdE,
                'median': self.Median,
                'Q1': self.Q1,
                'Q3': self.Q3,
                'IQR': self.IQR,
                'fileanomalyIQR': self.FileAnomalyIQR}

class FileInformation:
    def __init__(self, word_list, file_name):
        """
        takes a WordList of a file and the file name of that file to give
        statistics of that particular file
        :param word_list: a dictionary map word to word count representing the
                word count of particular file
        :param file_name: the file name of that file
        """

        # initialize
        num_word = len(word_list)
        total_word_count = sum(word_list.values())
        # 1 standard error analysis
        average_word_count = total_word_count / num_word
        # calculate the StdE
        std_err_word_count = 0
        for word_count in list(word_list.values()):
            std_err_word_count += (word_count - average_word_count) ** 2
        std_err_word_count /= num_word
        std_err_word_count = sqrt(std_err_word_count)

        # 2 iqr analysis
        temp_list = sorted(list(word_list.items()), key=itemgetter(1))
        mid = temp_list[int(num_word / 2)][1]
        q3 = temp_list[int(num_word * 3 / 4)][1]
        q1 = temp_list[int(num_word / 4)][1]
        iqr = q3 - q1

        # pack the data
        self.file_name = file_name
        self.num_word = num_word
        self.total_word_count = total_word_count
        self.word_count = word_list
        self.average = average_word_count
        self.std_err = std_err_word_count
        self.q1 = q1
        self.median = mid
        self.q3 = q3
        self.iqr = iqr
        self.hapax = (list(word_list.values()).count(1))

    def list_stat(self):
        """
        print all the statistics in a good manner

        """

        print()
        print('information for', "'" + self.file_name + "'")
        print('total word count:', self.total_word_count)
        print('1. in term of word count:')
        print('    average:', self.average, ' standard error:', self.std_err)
        print('    median:', self.median, ' Q1:', self.q1, ' Q3:', self.q3,
              ' IQR', self.iqr)
        print('2. in term of probability')
        print(
            '    average:', self.average / self.total_word_count,
            ' standard error:', self.std_err / self.total_word_count)
        print(
            '    median:', self.median / self.total_word_count,
            ' Q1:', self.q1 / self.total_word_count,
            ' Q3:', self.q3 / self.total_word_count,
            ' IQR', self.iqr / self.total_word_count)

    def plot(self, path, num_bins=0):
        """
        draw a histogram to represent the data
        :param num_bins: number of bars, default is
                    (Number different word in the file )/ 2,
                    if it is too large take 50 as default
                    (see '#default of num_bins')
        """
        # plot data
        mu = self.average  # mean of distribution
        sigma = self.std_err  # standard deviation of distribution
        if num_bins == 0:  # default of num_bins
            num_bins = min([round(self.num_word / 2), 50])
            # print num_bins
        # the histogram of the data
        n, bins, patches = plt.hist(
            list(self.word_count.values()), num_bins, normed=1,
            facecolor='green', alpha=0.5
        )

        # add a 'best fit' line
        y = mlab.normpdf(bins, mu, sigma)
        plt.plot(bins, y, 'r--')
        plt.xlabel('Word Count')
        plt.ylabel('Probability(how many words have this word count)')
        plt.title(r'Histogram of word count: $\mu=' +
                  str(self.average) + '$, $\sigma=' + str(self.std_err) + '$')

        # Tweak spacing to prevent clipping of ylabel
        plt.subplots_adjust(left=0.15)
        plt.savefig(path)
        plt.close()

    def return_statistics(self):
        """
        :return: a dictionary map the statistic name to the actual statistics

        """
        return {'name': self.file_name,
                'numUniqueWords': int(self.num_word),
                'totalwordCount': int(round(self.total_word_count, 2)),
                'median': self.median,
                'Q1': self.q1,
                'Q3': self.q3,
                'IQR': self.iqr,
                'average': truncate(self.average, 2),
                'stdE': self.std_err,
                'Hapax': self.hapax}
<commit_msg>Attempt to solve known bugs by using numpy<commit_after># -*- coding: utf-8 -*-

import matplotlib.pyplot as plt
import numpy as np
from matplotlib import mlab

def truncate(x, d):
    return int(x * (10.0 ** d)) / (10.0 ** d)

class CorpusInformation:
    def __init__(self, word_lists, l_files):
        """
        Converts word lists completely to statistic.

        Also gives anomalies about the files size.
        :param word_lists: an array contain dictionaries map from word to word
                           count each dictionary is word count of a particular
                           file.
        :param l_files: an parallel array of WordLists, contain file name of
                        the files(in order to plot).
        """
        # initialize
        num_file = len(word_lists)
        file_anomaly_std_err = {}
        file_anomaly_iqr = {}
        file_sizes = {}

        for i in range(num_file):
            file_sizes.update({l_files[i]: sum(word_lists[i].values())})
        file_sizes_list = list(file_sizes.values())

        # 1 standard error analysis
        average_file_size = np.mean(file_sizes_list)
        # Calculate the standard deviation
        std_file_size = np.std(file_sizes_list)

        # average_file_size = sum(file_sizes.values()) / num_file
        # std_err_file_size = 0
        # for file_size in list(file_sizes.values()):
        #     std_err_file_size += (file_size - average_file_size) ** 2
        # std_err_file_size /= num_file
        # std_err_file_size = sqrt(std_err_file_size)

        # Calculate the anomaly
        for file in l_files:
            if file_sizes[file] > average_file_size + 2 * std_file_size:
                file_anomaly_std_err.update({file.name: 'large'})
            elif file_sizes[file] < average_file_size - 2 * std_file_size:
                file_anomaly_std_err.update({file.name: 'small'})

        # 2 iqr analysis
        mid = np.median(file_sizes_list)
        q1 = np.percentile(file_sizes_list, 25)
        q3 = np.percentile(file_sizes_list, 75)
        iqr = q3 - q1

        # temp_list = sorted(list(file_sizes.items()), key=itemgetter(1))
        # temp_list = sorted(list(file_sizes.values()))
        # mid = temp_list[int(num_file / 2)]
        # q1 = temp_list[int(num_file / 4)]
        # q3 = temp_list[int(num_file * 3 / 4)]

        # calculate the anomaly
        for file in l_files:
            if file_sizes[file] > mid + 1.5 * iqr:
                file_anomaly_iqr.update({file.name: 'large'})
            elif file_sizes[file] < mid - 1.5 * iqr:
                file_anomaly_iqr.update({file.name: 'small'})

        # Pack the data
        self.NumFile = num_file  # number of files
        # an array of the total word count of each file
        self.FileSizes = file_sizes
        self.Average = average_file_size  # average file size
        self.StdE = std_file_size  # standard error of file size
        # an array contain dictionary map anomaly file to
        # how they are different from others(too large or too small)
        # analyzed in using standard error
        self.FileAnomalyStdE = file_anomaly_std_err
        self.Q1 = q1  # First quartile of all file sizes
        self.Median = mid  # Median of all file sizes
        self.Q3 = q3  # Third quartile of all file sizes
        self.IQR = iqr  # Interquartile range of all file sizes
        # an array contains dictionary map anomaly file to how they are
        # different from others(too large or too small) analyzed by using iqr
        self.FileAnomalyIQR = file_anomaly_iqr

    def list_stats(self):
        """Print all the statistics in a good manner."""
        print()
        print('average:', self.Average, ' standard error:', self.StdE)
        print('document size anomaly calculated using standard error:',
              self.FileAnomalyStdE)
        print('median:', self.Median, ' Q1:', self.Q1, ' Q3:', self.Q3,
              ' IQR', self.IQR)
        print('document size anomaly calculated using IQR:',
              self.FileAnomalyIQR)

    def plot(self, path):
        """Plot a bar chart to represent the statistics.

        x is the file name and y is the file size, represented by word count.
        """
        plt.bar(list(range(self.NumFile)), list(self.FileSizes.values()),
                align='center')
        plt.xticks(list(range(self.NumFile)), list(self.FileSizes.keys()))
        plt.xticks(rotation=50)
        plt.xlabel('File Name')
        plt.ylabel('File Size(in term of word count)')
        plt.savefig(path)
        plt.close()

    def return_statistics(self):
        """
        :return: a dictionary map the statistic name to the actual statistics
        """
        return {'average': truncate(self.Average, 3),
                'StdE': self.StdE,
                'fileanomalyStdE': self.FileAnomalyStdE,
                'median': self.Median,
                'Q1': self.Q1,
                'Q3': self.Q3,
                'IQR': self.IQR,
                'fileanomalyIQR': self.FileAnomalyIQR}

class FileInformation:
    def __init__(self, word_list, file_name):
        """Gives statistics of a particular file in a given file list

        :param word_list: a dictionary map word to word count representing the
                          word count of particular file
        :param file_name: the file name of that file
        """

        # initialize
        num_word = len(word_list)
        word_list_values = list(word_list.values())
        total_word_count = sum(word_list_values)
        # 1 standard error analysis
        average_word_count = total_word_count/num_word
        # calculate the StdE
        std_word_count = np.std(word_list_values)

        # std_err_word_count = 0
        # for word_count in list(word_list.values()):
        #     std_err_word_count += (word_count - average_word_count) ** 2
        # std_err_word_count /= num_word
        # std_err_word_count = sqrt(std_err_word_count)
        # 2 iqr analysis
        # temp_list = sorted(list(word_list.items()), key=itemgetter(1))
        # mid = temp_list[int(num_word / 2)][1]
        # q3 = temp_list[int(num_word * 3 / 4)][1]
        # q1 = temp_list[int(num_word / 4)][1]
        # iqr = q3 - q1

        # 2 iqr analysis
        mid = np.median(word_list_values)
        q1 = np.percentile(word_list_values, 25)
        q3 = np.percentile(word_list_values, 75)

        # pack the data
        self.file_name = file_name
        self.num_word = num_word
        self.total_word_count = total_word_count
        self.word_count = word_list
        self.average = average_word_count
        self.std_err = std_word_count
        self.q1 = q1
        self.median = mid
        self.q3 = q3
        self.iqr = q3 - q1
        self.hapax = (list(word_list.values()).count(1))

    def list_stat(self):
        """Print all the statistics in a good manner."""

        print()
        print('information for', "'" + self.file_name + "'")
        print('total word count:', self.total_word_count)
        print('1. in term of word count:')
        print('    average:', self.average, ' standard error:', self.std_err)
        print('    median:', self.median, ' Q1:', self.q1, ' Q3:', self.q3,
              ' IQR', self.iqr)
        print('2. in term of probability')
        print(
            '    average:', self.average / self.total_word_count,
            ' standard error:', self.std_err / self.total_word_count)
        print(
            '    median:', self.median / self.total_word_count,
            ' Q1:', self.q1 / self.total_word_count,
            ' Q3:', self.q3 / self.total_word_count,
            ' IQR', self.iqr / self.total_word_count)

    def plot(self, path, num_bins=0):
        """draw a histogram to represent the data

        :param num_bins: number of bars, default is
                         (Number different word in the file )/ 2, if it is too
                         large take 50 as default (see '#default of num_bins')
        """

        # plot data
        mu = self.average  # mean of distribution
        sigma = self.std_err  # standard deviation of distribution
        if num_bins == 0:  # default of num_bins
            num_bins = min([round(self.num_word / 2), 50])
            # print num_bins
        # the histogram of the data
        n, bins, patches = plt.hist(list(self.word_count.values()), num_bins,
                                    normed=1, facecolor='green', alpha=0.5)

        # add a 'best fit' line
        y = mlab.normpdf(bins, mu, sigma)
        plt.plot(bins, y, 'r--')
        plt.xlabel('Word Count')
        plt.ylabel('Probability(how many words have this word count)')
        plt.title(r'Histogram of word count: $\mu=' + str(self.average) +
                  '$, $\sigma=' + str(self.std_err) + '$')

        # Tweak spacing to prevent clipping of ylabel
        plt.subplots_adjust(left=0.15)
        plt.savefig(path)
        plt.close()

    def return_statistics(self):
        """
        :return: a dictionary map the statistic name to the actual statistics
        """
        return {'name': self.file_name,
                'numUniqueWords': int(self.num_word),
                'totalwordCount': int(round(self.total_word_count, 2)),
                'median': self.median,
                'Q1': self.q1,
                'Q3': self.q3,
                'IQR': self.iqr,
                'average': truncate(self.average, 2),
                'stdE': self.std_err,
                'Hapax': self.hapax}
<|endoftext|>


<commit_before>from nbconvert.preprocessors import ExecutePreprocessor
from traitlets import Bool, List

from . import NbGraderPreprocessor

class Execute(NbGraderPreprocessor, ExecutePreprocessor):

    interrupt_on_timeout = Bool(True)
    allow_errors = Bool(True)
    extra_arguments = List(["--HistoryManager.hist_file=:memory:"])
<commit_msg>Change options so other kernels work with nbgrader<commit_after>from nbconvert.preprocessors import ExecutePreprocessor
from traitlets import Bool, List
from textwrap import dedent

from . import NbGraderPreprocessor

class Execute(NbGraderPreprocessor, ExecutePreprocessor):

    interrupt_on_timeout = Bool(True)
    allow_errors = Bool(True)
    extra_arguments = List([], config=True, help=dedent(
        """
        A list of extra arguments to pass to the kernel. For python kernels,
        this defaults to ``--HistoryManager.hist_file=:memory:``. For other
        kernels this is just an empty list.
        """))

    def preprocess(self, nb, resources):
        kernel_name = nb.metadata.get('kernelspec', {}).get('name', 'python')
        if self.extra_arguments == [] and kernel_name == "python":
            self.extra_arguments = ["--HistoryManager.hist_file=:memory:"]

        return super(Execute, self).preprocess(nb, resources)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

# -*- encoding: utf-8 -*-

"""contextimpl.py: ContextImpl class that implements the Context interface
"""

import time
import os

import pulsar
import util
import InstanceCommunication_pb2

# For keeping track of accumulated metrics
class AccumulatedMetricDatum(object):
  def __init__(self):
    self.count = 0.0
    self.sum = 0.0
    self.max = float('-inf')
    self.min = float('inf')

  def record(self, value):
    self.count += 1
    self.sum += value
    if value > self.max:
      self.max = value
    if value < self.min:
      self.min = value

class ContextImpl(pulsar.Context):
  def __init__(self, instance_config, logger, pulsar_client, user_code, consumers):
    self.instance_config = instance_config
    self.log = logger
    self.pulsar_client = pulsar_client
    self.user_code_dir = os.path.dirname(user_code)
    self.consumers = consumers
    self.accumulated_metrics = {}
    self.publish_producers = {}
    self.publish_serializers = {}
    self.current_message_id = None
    self.current_topic_name = None
    self.current_start_time = None

  # Called on a per message basis to set the context for the current message
  def set_current_message_context(self, msgid, topic):
    self.current_message_id = msgid
    self.current_topic_name = topic
    self.current_start_time = time.time()

  def get_message_id(self):
    return self.current_message_id

  def get_topic_name(self):
    return self.current_topic_name

  def get_function_name(self):
    return self.instance_config.function_config.name

  def get_function_id(self):
    return self.instance_config.function_id

  def get_instance_id(self):
    return self.instance_config.instance_id

  def get_function_version(self):
    return self.instance_config.function_version

  def get_logger(self):
    return self.log

  def get_user_config_value(self, key):
    if key in self.instance_config.function_config.userConfig:
      return str(self.instance_config.function_config.userConfig[key])
    else:
      return None

  def get_user_config_map(self):
    return self.instance_config.function_config.userConfig

  def record_metric(self, metric_name, metric_value):
    if not metric_name in self.accumulated_metrics:
      self.accumulated_metrics[metric_name] = AccumulatedMetricDatum()
    self.accumulated_metrics[metric_name].update(metric_value)

  def get_output_topic(self):
    return self.instance_config.function_config.output

  def get_output_serde_class_name(self):
    return self.instance_config.function_config.outputSerdeClassName

  def publish(self, topic_name, message):
    return self.publish(topic_name, message, "serde.IdentitySerDe")

  def publish(self, topic_name, message, serde_class_name):
    if topic_name not in self.publish_producers:
      self.publish_producers[topic_name] = self.pulsar_client.create_producer(
        topic_name,
        block_if_queue_full=True,
        batching_enabled=True,
        batching_max_publish_delay_ms=1,
        max_pending_messages=100000
      )

    if serde_class_name not in self.publish_serializers:
      serde_klass = util.import_class(self.user_code_dir, serde_class_name)
      self.publish_serializers[serde_class_name] = serde_klass()

    output_bytes = self.publish_serializers[serde_class_name].serialize(message)
    self.publish_producers[topic_name].send_async(output_bytes, None)

  def ack(self, msgid, topic):
    if topic not in self.consumers:
      raise ValueError('Invalid topicname %s' % topic)
    self.consumers[topic].acknowledge(msgid)

  def get_and_reset_metrics(self):
    metrics = InstanceCommunication_pb2.MetricsData()
    for metric_name, accumulated_metric in self.accumulated_metrics.items():
      m = InstanceCommunication_pb2.MetricsData.DataDigest()
      m.count = accumulated_metric.count
      m.sum = accumulated_metric.sum
      m.max = accumulated_metric.max
      m.min = accumulated_metric.min
      metrics.metrics[metric_name] = m
    # TODO(sanjeev):- Make this thread safe
    self.accumulated_metrics.clear()
    return metrics
<commit_msg>Fix var names of context that were changed (#1676)<commit_after>#!/usr/bin/env python
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

# -*- encoding: utf-8 -*-

"""contextimpl.py: ContextImpl class that implements the Context interface
"""

import time
import os

import pulsar
import util
import InstanceCommunication_pb2

# For keeping track of accumulated metrics
class AccumulatedMetricDatum(object):
  def __init__(self):
    self.count = 0.0
    self.sum = 0.0
    self.max = float('-inf')
    self.min = float('inf')

  def record(self, value):
    self.count += 1
    self.sum += value
    if value > self.max:
      self.max = value
    if value < self.min:
      self.min = value

class ContextImpl(pulsar.Context):
  def __init__(self, instance_config, logger, pulsar_client, user_code, consumers):
    self.instance_config = instance_config
    self.log = logger
    self.pulsar_client = pulsar_client
    self.user_code_dir = os.path.dirname(user_code)
    self.consumers = consumers
    self.accumulated_metrics = {}
    self.publish_producers = {}
    self.publish_serializers = {}
    self.current_message_id = None
    self.current_topic_name = None
    self.current_start_time = None

  # Called on a per message basis to set the context for the current message
  def set_current_message_context(self, msgid, topic):
    self.current_message_id = msgid
    self.current_topic_name = topic
    self.current_start_time = time.time()

  def get_message_id(self):
    return self.current_message_id

  def get_topic_name(self):
    return self.current_topic_name

  def get_function_name(self):
    return self.instance_config.function_details.name

  def get_function_id(self):
    return self.instance_config.function_id

  def get_instance_id(self):
    return self.instance_config.instance_id

  def get_function_version(self):
    return self.instance_config.function_version

  def get_logger(self):
    return self.log

  def get_user_config_value(self, key):
    if key in self.instance_config.function_details.userConfig:
      return str(self.instance_config.function_details.userConfig[key])
    else:
      return None

  def get_user_config_map(self):
    return self.instance_config.function_details.userConfig

  def record_metric(self, metric_name, metric_value):
    if not metric_name in self.accumulated_metrics:
      self.accumulated_metrics[metric_name] = AccumulatedMetricDatum()
    self.accumulated_metrics[metric_name].update(metric_value)

  def get_output_topic(self):
    return self.instance_config.function_details.output

  def get_output_serde_class_name(self):
    return self.instance_config.function_details.outputSerdeClassName

  def publish(self, topic_name, message):
    return self.publish(topic_name, message, "serde.IdentitySerDe")

  def publish(self, topic_name, message, serde_class_name):
    if topic_name not in self.publish_producers:
      self.publish_producers[topic_name] = self.pulsar_client.create_producer(
        topic_name,
        block_if_queue_full=True,
        batching_enabled=True,
        batching_max_publish_delay_ms=1,
        max_pending_messages=100000
      )

    if serde_class_name not in self.publish_serializers:
      serde_klass = util.import_class(self.user_code_dir, serde_class_name)
      self.publish_serializers[serde_class_name] = serde_klass()

    output_bytes = self.publish_serializers[serde_class_name].serialize(message)
    self.publish_producers[topic_name].send_async(output_bytes, None)

  def ack(self, msgid, topic):
    if topic not in self.consumers:
      raise ValueError('Invalid topicname %s' % topic)
    self.consumers[topic].acknowledge(msgid)

  def get_and_reset_metrics(self):
    metrics = InstanceCommunication_pb2.MetricsData()
    for metric_name, accumulated_metric in self.accumulated_metrics.items():
      m = InstanceCommunication_pb2.MetricsData.DataDigest()
      m.count = accumulated_metric.count
      m.sum = accumulated_metric.sum
      m.max = accumulated_metric.max
      m.min = accumulated_metric.min
      metrics.metrics[metric_name] = m
    # TODO(sanjeev):- Make this thread safe
    self.accumulated_metrics.clear()
    return metrics
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Generates a list of hashes of EV certificates found in a log."""

import json
import os
import requests
import shutil
import StringIO
import sys
import tempfile
import urlparse
import zipfile

import generate_ev_whitelist
import golomb_code

import gflags

CRX_URL = (
    "https://clients2.googleusercontent.com/crx/blobs/QgAAAC6zw0qH2DJtnXe8Z7rUJP2bOoos30wQnFlfu4JmGbv_WafDN7cDNxLuif0rowVfBuLBdFtlTWFMKISh9OEN-IYAS8rlCeqrj26GNKznEPlFAMZSmuXhKxGN1KntVEK_KKCwwrfdp1cRxw/oafdbfcohdcjandcenmccfopbeklnicplive_main.platform.all.crx")
TRUNCATED_HASH_LENGTH = 8
GOLOMB_M_PARAMETER = 2 ** 47
NUM_FETCHING_PROCESSES = 40
LOGS_LIST = [
    "https://ct.googleapis.com/pilot",
    "https://ct.googleapis.com/aviator",
    "https://ct1.digicert-ct.com/log",
]

FLAGS = gflags.FLAGS

gflags.DEFINE_string(
    "output_crx", "chrome_ev_whitelist_%d",
    "Output Chrome extension. "
    "This zip file should be uploaded to the Chrome web store.")

gflags.DEFINE_string("output_truncated_hashes", "raw_truncated_hashes",
                     "The truncated hashes, hex-encoded, before compression.")

class EVCRXHandler(object):
    """Produces updated EV whitelist Chrome extension."""
    def __init__(self, crx_url=CRX_URL):
        req = requests.get(crx_url)
        if not req.ok:
            raise Exception("Error while fetching %s: %d" %
                            crx_url, req.status_code)
        # Skip the CRX header.
        crx_zip_bytes = req.content[566:]
        sio = StringIO.StringIO(crx_zip_bytes)
        zipped_crx = zipfile.ZipFile(sio)
        unpacked_crx_dir = tempfile.mkdtemp(prefix="tmpcrx")
        zipped_crx.extractall(unpacked_crx_dir)
        self._crx_dir = unpacked_crx_dir

    def _read_manifest_json(self):
        """Reads and parses the extension's metadata from manifest.json"""
        with open(os.path.join(self._crx_dir, "manifest.json")) as manifest:
            return json.load(manifest)

    def _write_manifest_json(self, json_to_write):
        """Overwrites the extension's metadata with the provided json."""
        with open(os.path.join(self._crx_dir, "manifest.json"), "wb") as manifest:
            json.dump(json_to_write, manifest)

    def manifest_version(self):
        """Returns the version number from the extension's metadata."""
        manifest = self._read_manifest_json()
        return int(manifest["version"])

    def update_ev_whitelist(self, compressed_ev_whitelist):
        """Writes an updated EV whitelist into the extension.

        Each new EV whitelist must be uploaded with a version number greater
        than the current version, so increments the version number.
        """
        manifest = self._read_manifest_json()
        manifest["version"] = str(self.manifest_version() + 1)
        self._write_manifest_json(manifest)
        with open(
            os.path.join(
                self._crx_dir,
                "_platform_specific",
                "all",
                "ev_hashes_whitelist.bin"),
            "wb") as hashes_file:
            hashes_file.write(compressed_ev_whitelist)

    def pack_to(self, dest_file_template):
        """Packs the extesion to the provided destination.

        Note that the provided dest_file_template is a template string so
        the version number can be filled in.
        Returns the exact file name that was written.
        """
        fname = dest_file_template % self.manifest_version()
        shutil.make_archive(fname, "zip", self._crx_dir)
        return fname

    def cleanup(self):
        """Deletes the directory containing the unpacked crx."""
        shutil.rmtree(self._crx_dir)
        self._crx_dir = None

def generate_new_ev_hashes_whitelist(logs_list):
    """Scans the provided logs_list and generates a compressed EV certs
    whitelist for all EV certs in all the logs."""
    all_hashes_set = set()
    for log_url in logs_list:
        output_dir = None
        if FLAGS.output_directory:
            parsed_url = urlparse.urlsplit(log_url)
            output_dir = os.path.join(
                FLAGS.output_directory,
                parsed_url.netloc,
                parsed_url.path.strip("/"))
        print "Scanning log %s, certificates will be in %s" % (
            log_url, output_dir)
        res, hashes_set = (
            generate_ev_whitelist.generate_ev_cert_hashes_from_log(
                log_url,
                FLAGS.multi,
                output_dir))
        print "Scanned %d, %d matched, %d failed parsing (strict/partial)" % (
            res.total, res.matches, res.errors)
        print "There are %d EV hashes." % (len(hashes_set))
        old_hashes_count = len(all_hashes_set)
        all_hashes_set = all_hashes_set.union(hashes_set)
        print "%d new EV certificates." % (
            (len(all_hashes_set) - old_hashes_count))

    hashes_list = list(all_hashes_set)
    hashes_list.sort()
    with open(FLAGS.output_truncated_hashes, "wb") as hashes_file:
        for trimmed_hash in hashes_list:
            hashes_file.write(trimmed_hash)
    golomb_coded_bytes = golomb_code.golomb_encode(
        hashes_list, FLAGS.hash_length, 2 ** FLAGS.two_power)
    return golomb_coded_bytes

def main():
    crx = EVCRXHandler()
    print "Current manifest version is %d." % (crx.manifest_version())
    compressed_hashes = generate_new_ev_hashes_whitelist(LOGS_LIST)
    crx.update_ev_whitelist(compressed_hashes)
    output_file = crx.pack_to(FLAGS.output_crx)
    print "New EV whitelist is at %s" % (output_file)
    crx.cleanup()

if __name__ == "__main__":
    sys.argv = FLAGS(sys.argv)
    main()
<commit_msg>EV whitelist: Gather EV certs from all pending logs<commit_after>#!/usr/bin/env python
"""Generates a list of hashes of EV certificates found in a log."""

import json
import os
import requests
import shutil
import StringIO
import sys
import tempfile
import urlparse
import zipfile

import generate_ev_whitelist
import golomb_code

import gflags

CRX_URL = (
    "https://clients2.googleusercontent.com/crx/blobs/QgAAAC6zw0qH2DJtnXe8Z7rUJP2bOoos30wQnFlfu4JmGbv_WafDN7cDNxLuif0rowVfBuLBdFtlTWFMKISh9OEN-IYAS8rlCeqrj26GNKznEPlFAMZSmuXhKxGN1KntVEK_KKCwwrfdp1cRxw/oafdbfcohdcjandcenmccfopbeklnicplive_main.platform.all.crx")
TRUNCATED_HASH_LENGTH = 8
GOLOMB_M_PARAMETER = 2 ** 47
NUM_FETCHING_PROCESSES = 40
LOGS_LIST = [
    "https://ct.googleapis.com/pilot",
    "https://ct.googleapis.com/aviator",
    "https://ct1.digicert-ct.com/log",
    "https://log.certly.io",
    "https://ct.izenpe.com",
]

FLAGS = gflags.FLAGS

gflags.DEFINE_string(
    "output_crx", "chrome_ev_whitelist_%d",
    "Output Chrome extension. "
    "This zip file should be uploaded to the Chrome web store.")

gflags.DEFINE_string("output_truncated_hashes", "raw_truncated_hashes",
                     "The truncated hashes, hex-encoded, before compression.")

class EVCRXHandler(object):
    """Produces updated EV whitelist Chrome extension."""
    def __init__(self, crx_url=CRX_URL):
        req = requests.get(crx_url)
        if not req.ok:
            raise Exception("Error while fetching %s: %d" %
                            crx_url, req.status_code)
        # Skip the CRX header.
        crx_zip_bytes = req.content[566:]
        sio = StringIO.StringIO(crx_zip_bytes)
        zipped_crx = zipfile.ZipFile(sio)
        unpacked_crx_dir = tempfile.mkdtemp(prefix="tmpcrx")
        zipped_crx.extractall(unpacked_crx_dir)
        self._crx_dir = unpacked_crx_dir

    def _read_manifest_json(self):
        """Reads and parses the extension's metadata from manifest.json"""
        with open(os.path.join(self._crx_dir, "manifest.json")) as manifest:
            return json.load(manifest)

    def _write_manifest_json(self, json_to_write):
        """Overwrites the extension's metadata with the provided json."""
        with open(os.path.join(self._crx_dir, "manifest.json"), "wb") as manifest:
            json.dump(json_to_write, manifest)

    def manifest_version(self):
        """Returns the version number from the extension's metadata."""
        manifest = self._read_manifest_json()
        return int(manifest["version"])

    def update_ev_whitelist(self, compressed_ev_whitelist):
        """Writes an updated EV whitelist into the extension.

        Each new EV whitelist must be uploaded with a version number greater
        than the current version, so increments the version number.
        """
        manifest = self._read_manifest_json()
        manifest["version"] = str(self.manifest_version() + 1)
        self._write_manifest_json(manifest)
        with open(
            os.path.join(
                self._crx_dir,
                "_platform_specific",
                "all",
                "ev_hashes_whitelist.bin"),
            "wb") as hashes_file:
            hashes_file.write(compressed_ev_whitelist)

    def pack_to(self, dest_file_template):
        """Packs the extesion to the provided destination.

        Note that the provided dest_file_template is a template string so
        the version number can be filled in.
        Returns the exact file name that was written.
        """
        fname = dest_file_template % self.manifest_version()
        shutil.make_archive(fname, "zip", self._crx_dir)
        return fname

    def cleanup(self):
        """Deletes the directory containing the unpacked crx."""
        shutil.rmtree(self._crx_dir)
        self._crx_dir = None

def generate_new_ev_hashes_whitelist(logs_list):
    """Scans the provided logs_list and generates a compressed EV certs
    whitelist for all EV certs in all the logs."""
    all_hashes_set = set()
    for log_url in logs_list:
        output_dir = None
        if FLAGS.output_directory:
            parsed_url = urlparse.urlsplit(log_url)
            output_dir = os.path.join(
                FLAGS.output_directory,
                parsed_url.netloc,
                parsed_url.path.strip("/"))
        print "Scanning log %s, certificates will be in %s" % (
            log_url, output_dir)
        res, hashes_set = (
            generate_ev_whitelist.generate_ev_cert_hashes_from_log(
                log_url,
                FLAGS.multi,
                output_dir))
        print "Scanned %d, %d matched, %d failed parsing (strict/partial)" % (
            res.total, res.matches, res.errors)
        print "There are %d EV hashes." % (len(hashes_set))
        old_hashes_count = len(all_hashes_set)
        all_hashes_set = all_hashes_set.union(hashes_set)
        print "%d new EV certificates." % (
            (len(all_hashes_set) - old_hashes_count))

    hashes_list = list(all_hashes_set)
    hashes_list.sort()
    with open(FLAGS.output_truncated_hashes, "wb") as hashes_file:
        for trimmed_hash in hashes_list:
            hashes_file.write(trimmed_hash)
    golomb_coded_bytes = golomb_code.golomb_encode(
        hashes_list, FLAGS.hash_length, 2 ** FLAGS.two_power)
    return golomb_coded_bytes

def main():
    crx = EVCRXHandler()
    print "Current manifest version is %d." % (crx.manifest_version())
    compressed_hashes = generate_new_ev_hashes_whitelist(LOGS_LIST)
    crx.update_ev_whitelist(compressed_hashes)
    output_file = crx.pack_to(FLAGS.output_crx)
    print "New EV whitelist is at %s" % (output_file)
    crx.cleanup()

if __name__ == "__main__":
    sys.argv = FLAGS(sys.argv)
    main()
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2019 HuggingFace Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, division, print_function, unicode_literals

import os
import sys
from io import open
import tempfile
import shutil
import unittest

if sys.version_info[0] == 2:
    import cPickle as pickle

    class TemporaryDirectory(object):
        """Context manager for tempfile.mkdtemp() so it's usable with "with" statement."""
        def __enter__(self):
            self.name = tempfile.mkdtemp()
            return self.name
        def __exit__(self, exc_type, exc_value, traceback):
            shutil.rmtree(self.name)
else:
    import pickle
    TemporaryDirectory = tempfile.TemporaryDirectory
    unicode = str

class CommonTestCases:

    class CommonTokenizerTester(unittest.TestCase):

        tokenizer_class = None

        def setUp(self):
            self.tmpdirname = tempfile.mkdtemp()

        def tearDown(self):
            shutil.rmtree(self.tmpdirname)

        def get_tokenizer(self, **kwargs):
            raise NotImplementedError

        def get_input_output_texts(self):
            raise NotImplementedError

        def test_tokenizers_common_properties(self):
            tokenizer = self.get_tokenizer()
            attributes_list = ["bos_token", "eos_token", "unk_token", "sep_token",
                                "pad_token", "cls_token", "mask_token"]
            for attr in attributes_list:
                self.assertTrue(hasattr(tokenizer, attr))
                self.assertTrue(hasattr(tokenizer, attr + "_id"))

            self.assertTrue(hasattr(tokenizer, "additional_special_tokens"))
            self.assertTrue(hasattr(tokenizer, 'additional_special_tokens_ids'))

            attributes_list = ["max_len", "init_inputs", "init_kwargs", "added_tokens_encoder",
                                "added_tokens_decoder"]
            for attr in attributes_list:
                self.assertTrue(hasattr(tokenizer, attr))

        def test_save_and_load_tokenizer(self):
            # safety check on max_len default value so we are sure the test works
            tokenizer = self.get_tokenizer()
            self.assertNotEqual(tokenizer.max_len, 42)

            # Now let's start the test
            tokenizer = self.get_tokenizer(max_len=42)

            before_tokens = tokenizer.encode(u"He is very happy, UNwant\u00E9d,running")

            with TemporaryDirectory() as tmpdirname:
                tokenizer.save_pretrained(tmpdirname)
                tokenizer = self.tokenizer_class.from_pretrained(tmpdirname)

                after_tokens = tokenizer.encode(u"He is very happy, UNwant\u00E9d,running")
                self.assertListEqual(before_tokens, after_tokens)

                self.assertEqual(tokenizer.max_len, 42)
                tokenizer = self.tokenizer_class.from_pretrained(tmpdirname, max_len=43)
                self.assertEqual(tokenizer.max_len, 43)

        def test_pickle_tokenizer(self):
            tokenizer = self.get_tokenizer()
            self.assertIsNotNone(tokenizer)

            text = u"Munich and Berlin are nice cities"
            subwords = tokenizer.tokenize(text)

            with TemporaryDirectory() as tmpdirname:

                filename = os.path.join(tmpdirname, u"tokenizer.bin")
                pickle.dump(tokenizer, open(filename, "wb"))

                tokenizer_new = pickle.load(open(filename, "rb"))

            subwords_loaded = tokenizer_new.tokenize(text)

            self.assertListEqual(subwords, subwords_loaded)

        def test_add_tokens_tokenizer(self):
            tokenizer = self.get_tokenizer()

            vocab_size = tokenizer.vocab_size
            all_size = len(tokenizer)

            self.assertNotEqual(vocab_size, 0)
            self.assertEqual(vocab_size, all_size)

            new_toks = ["aaaaa bbbbbb", "cccccccccdddddddd"]
            added_toks = tokenizer.add_tokens(new_toks)
            vocab_size_2 = tokenizer.vocab_size
            all_size_2 = len(tokenizer)

            self.assertNotEqual(vocab_size_2, 0)
            self.assertEqual(vocab_size, vocab_size_2)
            self.assertEqual(added_toks, len(new_toks))
            self.assertEqual(all_size_2, all_size + len(new_toks))

            tokens = tokenizer.encode("aaaaa bbbbbb low cccccccccdddddddd l")
            out_string = tokenizer.decode(tokens)

            self.assertGreaterEqual(len(tokens), 4)
            self.assertGreater(tokens[0], tokenizer.vocab_size - 1)
            self.assertGreater(tokens[-2], tokenizer.vocab_size - 1)

            new_toks_2 = {'eos_token': ">>>>|||<||<<|<<",
                          'pad_token': "<<<<<|||>|>>>>|>"}
            added_toks_2 = tokenizer.add_special_tokens(new_toks_2)
            vocab_size_3 = tokenizer.vocab_size
            all_size_3 = len(tokenizer)

            self.assertNotEqual(vocab_size_3, 0)
            self.assertEqual(vocab_size, vocab_size_3)
            self.assertEqual(added_toks_2, len(new_toks_2))
            self.assertEqual(all_size_3, all_size_2 + len(new_toks_2))

            tokens = tokenizer.encode(">>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l")
            out_string = tokenizer.decode(tokens)

            self.assertGreaterEqual(len(tokens), 6)
            self.assertGreater(tokens[0], tokenizer.vocab_size - 1)
            self.assertGreater(tokens[0], tokens[1])
            self.assertGreater(tokens[-2], tokenizer.vocab_size - 1)
            self.assertGreater(tokens[-2], tokens[-3])
            self.assertEqual(tokens[0], tokenizer.eos_token_id)
            self.assertEqual(tokens[-2], tokenizer.pad_token_id)

        def test_required_methods_tokenizer(self):
            tokenizer = self.get_tokenizer()
            input_text, output_text = self.get_input_output_texts()

            tokens = tokenizer.tokenize(input_text)
            ids = tokenizer.convert_tokens_to_ids(tokens)
            ids_2 = tokenizer.encode(input_text)
            self.assertListEqual(ids, ids_2)

            tokens_2 = tokenizer.convert_ids_to_tokens(ids)
            text_2 = tokenizer.decode(ids)

            self.assertEqual(text_2, output_text)

            self.assertNotEqual(len(tokens_2), 0)
            self.assertIsInstance(text_2, (str, unicode))

        def test_pretrained_model_lists(self):
            weights_list = list(self.tokenizer_class.max_model_input_sizes.keys())
            weights_lists_2 = []
            for file_id, map_list in self.tokenizer_class.pretrained_vocab_files_map.items():
                weights_lists_2.append(list(map_list.keys()))

            for weights_list_2 in weights_lists_2:
                self.assertListEqual(weights_list, weights_list_2)

        def test_mask_output(self):
            if sys.version_info <= (3, 0):
                return

            tokenizer = self.get_tokenizer()

            if tokenizer.add_special_tokens_sentences_pair.__qualname__.split('.')[0] != "PreTrainedTokenizer":
                seq_0 = "Test this method."
                seq_1 = "With these inputs."
                sequences, mask = tokenizer.encode(seq_0, seq_1, add_special_tokens=True, output_mask=True)
                assert len(sequences) == len(mask)

        def test_number_of_added_tokens(self):
            tokenizer = self.get_tokenizer()

            seq_0 = "Test this method."
            seq_1 = "With these inputs."

            sequences = tokenizer.encode(seq_0, seq_1)
            attached_sequences = tokenizer.encode(seq_0, seq_1, add_special_tokens=True)

            # Method is implemented (e.g. not GPT-2)
            if len(attached_sequences) != 2:
                assert tokenizer.num_added_tokens(pair=True) == len(attached_sequences) - sum([len(seq) for seq in sequences])

        def test_maximum_encoding_length_single_input(self):
            tokenizer = self.get_tokenizer()

            seq_0 = "This is a sentence to be encoded."

            sequence = tokenizer.encode(seq_0)
            num_added_tokens = tokenizer.num_added_tokens()
            total_length = len(sequence) + num_added_tokens
            truncated_sequence = tokenizer.encode(seq_0, max_length=total_length - 2, add_special_tokens=True)

            assert len(truncated_sequence) == total_length - 2
            assert truncated_sequence == tokenizer.add_special_tokens_single_sentence(sequence[:-2])

        def test_maximum_encoding_length_pair_input(self):
            tokenizer = self.get_tokenizer()

            seq_0 = "This is a sentence to be encoded."
            seq_1 = "This is another sentence to be encoded."

            sequence = tokenizer.encode(seq_0, seq_1, add_special_tokens=True)
            truncated_second_sequence = tokenizer.add_special_tokens_sentences_pair(
                tokenizer.encode(seq_0),
                tokenizer.encode(seq_1)[:-2]
            )
            truncated_sequence = tokenizer.encode(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True)

            assert len(truncated_sequence) == len(sequence) - 2
            assert truncated_sequence == truncated_second_sequence
<commit_msg>encode + encode_plus tests modified<commit_after># coding=utf-8
# Copyright 2019 HuggingFace Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, division, print_function, unicode_literals

import os
import sys
from io import open
import tempfile
import shutil
import unittest

if sys.version_info[0] == 2:
    import cPickle as pickle

    class TemporaryDirectory(object):
        """Context manager for tempfile.mkdtemp() so it's usable with "with" statement."""
        def __enter__(self):
            self.name = tempfile.mkdtemp()
            return self.name
        def __exit__(self, exc_type, exc_value, traceback):
            shutil.rmtree(self.name)
else:
    import pickle
    TemporaryDirectory = tempfile.TemporaryDirectory
    unicode = str

class CommonTestCases:

    class CommonTokenizerTester(unittest.TestCase):

        tokenizer_class = None

        def setUp(self):
            self.tmpdirname = tempfile.mkdtemp()

        def tearDown(self):
            shutil.rmtree(self.tmpdirname)

        def get_tokenizer(self, **kwargs):
            raise NotImplementedError

        def get_input_output_texts(self):
            raise NotImplementedError

        def test_tokenizers_common_properties(self):
            tokenizer = self.get_tokenizer()
            attributes_list = ["bos_token", "eos_token", "unk_token", "sep_token",
                                "pad_token", "cls_token", "mask_token"]
            for attr in attributes_list:
                self.assertTrue(hasattr(tokenizer, attr))
                self.assertTrue(hasattr(tokenizer, attr + "_id"))

            self.assertTrue(hasattr(tokenizer, "additional_special_tokens"))
            self.assertTrue(hasattr(tokenizer, 'additional_special_tokens_ids'))

            attributes_list = ["max_len", "init_inputs", "init_kwargs", "added_tokens_encoder",
                                "added_tokens_decoder"]
            for attr in attributes_list:
                self.assertTrue(hasattr(tokenizer, attr))

        def test_save_and_load_tokenizer(self):
            # safety check on max_len default value so we are sure the test works
            tokenizer = self.get_tokenizer()
            self.assertNotEqual(tokenizer.max_len, 42)

            # Now let's start the test
            tokenizer = self.get_tokenizer(max_len=42)

            before_tokens = tokenizer.encode(u"He is very happy, UNwant\u00E9d,running")

            with TemporaryDirectory() as tmpdirname:
                tokenizer.save_pretrained(tmpdirname)
                tokenizer = self.tokenizer_class.from_pretrained(tmpdirname)

                after_tokens = tokenizer.encode(u"He is very happy, UNwant\u00E9d,running")
                self.assertListEqual(before_tokens, after_tokens)

                self.assertEqual(tokenizer.max_len, 42)
                tokenizer = self.tokenizer_class.from_pretrained(tmpdirname, max_len=43)
                self.assertEqual(tokenizer.max_len, 43)

        def test_pickle_tokenizer(self):
            tokenizer = self.get_tokenizer()
            self.assertIsNotNone(tokenizer)

            text = u"Munich and Berlin are nice cities"
            subwords = tokenizer.tokenize(text)

            with TemporaryDirectory() as tmpdirname:

                filename = os.path.join(tmpdirname, u"tokenizer.bin")
                pickle.dump(tokenizer, open(filename, "wb"))

                tokenizer_new = pickle.load(open(filename, "rb"))

            subwords_loaded = tokenizer_new.tokenize(text)

            self.assertListEqual(subwords, subwords_loaded)

        def test_add_tokens_tokenizer(self):
            tokenizer = self.get_tokenizer()

            vocab_size = tokenizer.vocab_size
            all_size = len(tokenizer)

            self.assertNotEqual(vocab_size, 0)
            self.assertEqual(vocab_size, all_size)

            new_toks = ["aaaaa bbbbbb", "cccccccccdddddddd"]
            added_toks = tokenizer.add_tokens(new_toks)
            vocab_size_2 = tokenizer.vocab_size
            all_size_2 = len(tokenizer)

            self.assertNotEqual(vocab_size_2, 0)
            self.assertEqual(vocab_size, vocab_size_2)
            self.assertEqual(added_toks, len(new_toks))
            self.assertEqual(all_size_2, all_size + len(new_toks))

            tokens = tokenizer.encode("aaaaa bbbbbb low cccccccccdddddddd l")
            out_string = tokenizer.decode(tokens)

            self.assertGreaterEqual(len(tokens), 4)
            self.assertGreater(tokens[0], tokenizer.vocab_size - 1)
            self.assertGreater(tokens[-2], tokenizer.vocab_size - 1)

            new_toks_2 = {'eos_token': ">>>>|||<||<<|<<",
                          'pad_token': "<<<<<|||>|>>>>|>"}
            added_toks_2 = tokenizer.add_special_tokens(new_toks_2)
            vocab_size_3 = tokenizer.vocab_size
            all_size_3 = len(tokenizer)

            self.assertNotEqual(vocab_size_3, 0)
            self.assertEqual(vocab_size, vocab_size_3)
            self.assertEqual(added_toks_2, len(new_toks_2))
            self.assertEqual(all_size_3, all_size_2 + len(new_toks_2))

            tokens = tokenizer.encode(">>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l")
            out_string = tokenizer.decode(tokens)

            self.assertGreaterEqual(len(tokens), 6)
            self.assertGreater(tokens[0], tokenizer.vocab_size - 1)
            self.assertGreater(tokens[0], tokens[1])
            self.assertGreater(tokens[-2], tokenizer.vocab_size - 1)
            self.assertGreater(tokens[-2], tokens[-3])
            self.assertEqual(tokens[0], tokenizer.eos_token_id)
            self.assertEqual(tokens[-2], tokenizer.pad_token_id)

        def test_required_methods_tokenizer(self):
            tokenizer = self.get_tokenizer()
            input_text, output_text = self.get_input_output_texts()

            tokens = tokenizer.tokenize(input_text)
            ids = tokenizer.convert_tokens_to_ids(tokens)
            ids_2 = tokenizer.encode(input_text)
            self.assertListEqual(ids, ids_2)

            tokens_2 = tokenizer.convert_ids_to_tokens(ids)
            text_2 = tokenizer.decode(ids)

            self.assertEqual(text_2, output_text)

            self.assertNotEqual(len(tokens_2), 0)
            self.assertIsInstance(text_2, (str, unicode))

        def test_pretrained_model_lists(self):
            weights_list = list(self.tokenizer_class.max_model_input_sizes.keys())
            weights_lists_2 = []
            for file_id, map_list in self.tokenizer_class.pretrained_vocab_files_map.items():
                weights_lists_2.append(list(map_list.keys()))

            for weights_list_2 in weights_lists_2:
                self.assertListEqual(weights_list, weights_list_2)

        def test_mask_output(self):
            if sys.version_info <= (3, 0):
                return

            tokenizer = self.get_tokenizer()

            if tokenizer.add_special_tokens_sentences_pair.__qualname__.split('.')[0] != "PreTrainedTokenizer":
                seq_0 = "Test this method."
                seq_1 = "With these inputs."
                information = tokenizer.encode_plus(seq_0, seq_1, add_special_tokens=True, output_mask=True)
                sequences, mask = information["sequence"], information["mask"]
                assert len(sequences) == len(mask)

        def test_number_of_added_tokens(self):
            tokenizer = self.get_tokenizer()

            seq_0 = "Test this method."
            seq_1 = "With these inputs."

            sequences = tokenizer.encode(seq_0, seq_1)
            attached_sequences = tokenizer.encode(seq_0, seq_1, add_special_tokens=True)

            # Method is implemented (e.g. not GPT-2)
            if len(attached_sequences) != 2:
                assert tokenizer.num_added_tokens(pair=True) == len(attached_sequences) - len(sequences)

        def test_maximum_encoding_length_single_input(self):
            tokenizer = self.get_tokenizer()

            seq_0 = "This is a sentence to be encoded."

            sequence = tokenizer.encode(seq_0)
            num_added_tokens = tokenizer.num_added_tokens()
            total_length = len(sequence) + num_added_tokens
            information = tokenizer.encode_plus(seq_0, max_length=total_length - 2, add_special_tokens=True)

            truncated_sequence = information["sequence"]
            overflowing_tokens = information["overflowing_tokens"]

            assert len(overflowing_tokens) == 2
            assert len(truncated_sequence) == total_length - 2
            assert truncated_sequence == tokenizer.add_special_tokens_single_sentence(sequence[:-2])

        def test_maximum_encoding_length_pair_input(self):
            tokenizer = self.get_tokenizer()

            seq_0 = "This is a sentence to be encoded."
            seq_1 = "This is another sentence to be encoded."

            sequence = tokenizer.encode(seq_0, seq_1, add_special_tokens=True)
            truncated_second_sequence = tokenizer.add_special_tokens_sentences_pair(
                tokenizer.encode(seq_0),
                tokenizer.encode(seq_1)[:-2]
            )
            information = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True)

            truncated_sequence = information["sequence"]
            overflowing_tokens = information["overflowing_tokens"]

            assert len(truncated_sequence) == len(sequence) - 2
            assert truncated_sequence == truncated_second_sequence
<|endoftext|>


<commit_before>"""
Module which implements decoding functions using multiple attentions
for RNN decoders.

See http://arxiv.org/abs/1606.07481
"""
#tests: lint

import tensorflow as tf
import traceback

from neuralmonkey.logging import debug
from neuralmonkey.nn.projection import maxout, linear

# pylint: disable=too-many-arguments
# Great functions require great number of parameters
def attention_decoder(decoder_inputs, initial_state, attention_objects,
                      cell, maxout_size, loop_function=None, scope=None,
                      summary_collections=None):
    outputs = []
    states = []

    #### WTF does this do?
    # do manualy broadcasting of the initial state if we want it
    # to be the same for all inputs
    if len(initial_state.get_shape()) == 1:
        debug("Warning! I am doing this weird job.")
        state_size = initial_state.get_shape()[0].value
        initial_state = tf.reshape(tf.tile(initial_state,
                                           tf.shape(decoder_inputs[0])[:1]),
                                   [-1, state_size])

    with tf.variable_scope(scope or "attention_decoder"):
        output, state = decode_step(decoder_inputs[0], initial_state,
                                    attention_objects, cell, maxout_size)
        outputs.append(output)
        states.append(state)

        for step in range(1, len(decoder_inputs)):
            tf.get_variable_scope().reuse_variables()

            if loop_function:
                decoder_input = loop_function(output, step)
            else:
                decoder_input = decoder_inputs[step]

            output, state = decode_step(decoder_input, state, attention_objects,
                                        cell, maxout_size)
            outputs.append(output)
            states.append(state)

    if summary_collections:
        for i, a in enumerate(attention_objects):
            alignments = tf.expand_dims(tf.transpose(tf.pack(a.attentions_in_time), perm=[1, 2, 0]), -1)
            tf.image_summary("attention_{}".format(i), alignments,
                             collections=summary_collections, max_images=1000)

    return outputs, states

def decode_step(prev_output, prev_state, attention_objects,
                rnn_cell, maxout_size):
    """This function implements equations in section A.2.2 of the
    Bahdanau et al. (2015) paper, on pages 13 and 14.

    Arguments:
        prev_output: Previous decoded output (denoted by y_i-1)
        prev_state: Previous state (denoted by s_i-1)
        attention_objects: Objects that do attention
        rnn_cell: The RNN cell to use (should be GRU)
        maxout_size: The size of the maxout hidden layer (denoted by l)

    Returns:
        Tuple of the new output and state
    """
    ## compute c_i:
    contexts = [a.attention(prev_state) for a in attention_objects]

    # TODO dropouts??

    ## compute t_i:
    output = maxout([prev_state, prev_output] + contexts, maxout_size)

    ## compute s_i based on y_i-1, c_i and s_i-1
    _, state = rnn_cell(tf.concat(1, [prev_output] + contexts), prev_state)

    return output, state

class Attention(object):
    #pylint: disable=unused-argument,too-many-instance-attributes
    # For maintaining the same API as in CoverageAttention
    def __init__(self, attention_states, scope, dropout_placeholder,
                 input_weights=None, max_fertility=None):
        self.scope = scope
        self.attentions_in_time = []
        self.attention_states = tf.nn.dropout(attention_states,
                                              dropout_placeholder)
        self.input_weights = input_weights

        with tf.variable_scope(scope):
            self.attn_length = attention_states.get_shape()[1].value
            self.attn_size = attention_states.get_shape()[2].value

            # To calculate W1 * h_t we use a 1-by-1 convolution, need to reshape
            # before.
            self.att_states_reshaped = tf.reshape(
                self.attention_states,
                [-1, self.attn_length, 1, self.attn_size])

            self.attention_vec_size = self.attn_size    # Size of query vectors
                                                        # for attention.

            k = tf.get_variable(
                "AttnW",
                [1, 1, self.attn_size, self.attention_vec_size])

            self.hidden_features = tf.nn.conv2d(self.att_states_reshaped, k,
                                                [1, 1, 1, 1], "SAME")

            #pylint: disable=invalid-name
            # see comments on disabling invalid names below
            self.v = tf.get_variable("AttnV", [self.attention_vec_size])

    def attention(self, query_state):
        """Put attention masks on att_states_reshaped
           using hidden_features and query.
        """

        with tf.variable_scope(self.scope+"/Attention"):
            y = linear(query_state, self.attention_vec_size)
            y = tf.reshape(y, [-1, 1, 1, self.attention_vec_size])

            #pylint: disable=invalid-name
            # code copied from tensorflow. Suggestion: rename the variables
            # according to the Bahdanau paper
            s = self.get_logits(y)

            if self.input_weights is None:
                a = tf.nn.softmax(s)
            else:
                a_all = tf.nn.softmax(s) * self.input_weights
                norm = tf.reduce_sum(a_all, 1, keep_dims=True) + 1e-8
                a = a_all / norm

            self.attentions_in_time.append(a)

            # Now calculate the attention-weighted vector d.
            d = tf.reduce_sum(tf.reshape(a, [-1, self.attn_length, 1, 1])
                              * self.att_states_reshaped, [1, 2])

            return tf.reshape(d, [-1, self.attn_size])

    def get_logits(self, y):
        # Attention mask is a softmax of v^T * tanh(...).
        return tf.reduce_sum(self.v * tf.tanh(self.hidden_features + y), [2, 3])

class CoverageAttention(Attention):

    # pylint: disable=too-many-arguments
    # Great objects require great number of parameters
    def __init__(self, attention_states, scope, dropout_placeholder,
                 input_weights=None, max_fertility=5):

        super(CoverageAttention, self).__init__(attention_states, scope,
                                                dropout_placeholder,
                                                input_weights=input_weights,
                                                max_fertility=max_fertility)

        self.coverage_weights = tf.get_variable("coverage_matrix",
                                                [1, 1, 1, self.attn_size])
        self.fertility_weights = tf.get_variable("fertility_matrix",
                                                 [1, 1, self.attn_size])
        self.max_fertility = max_fertility

        self.fertility = 1e-8 + self.max_fertility * tf.sigmoid(
            tf.reduce_sum(self.fertility_weights * self.attention_states, [2]))

    def get_logits(self, y):
        coverage = sum(
            self.attentions_in_time) / self.fertility * self.input_weights

        logits = tf.reduce_sum(
            self.v * tf.tanh(
                self.hidden_features + y + self.coverage_weights * tf.reshape(
                    coverage, [-1, self.attn_length, 1, 1])),
            [2, 3])

        return logits
<commit_msg>Clean up<commit_after>"""
Module which implements decoding functions using multiple attentions
for RNN decoders.

See http://arxiv.org/abs/1606.07481
"""
#tests: lint

import tensorflow as tf

from neuralmonkey.logging import debug
from neuralmonkey.nn.projection import maxout, linear

# pylint: disable=too-many-arguments
# Great functions require great number of parameters
def attention_decoder(decoder_inputs, initial_state, attention_objects,
                      cell, maxout_size, loop_function=None, scope=None,
                      summary_collections=None):
    outputs = []
    states = []

    #### WTF does this do?
    # do manualy broadcasting of the initial state if we want it
    # to be the same for all inputs
    if len(initial_state.get_shape()) == 1:
        debug("Warning! I am doing this weird job.")
        state_size = initial_state.get_shape()[0].value
        initial_state = tf.reshape(tf.tile(initial_state,
                                           tf.shape(decoder_inputs[0])[:1]),
                                   [-1, state_size])

    with tf.variable_scope(scope or "attention_decoder"):
        output, state = decode_step(decoder_inputs[0], initial_state,
                                    attention_objects, cell, maxout_size)
        outputs.append(output)
        states.append(state)

        for step in range(1, len(decoder_inputs)):
            tf.get_variable_scope().reuse_variables()

            if loop_function:
                decoder_input = loop_function(output, step)
            else:
                decoder_input = decoder_inputs[step]

            output, state = decode_step(decoder_input, state, attention_objects,
                                        cell, maxout_size)
            outputs.append(output)
            states.append(state)

    if summary_collections:
        for i, a in enumerate(attention_objects):
            alignments = tf.expand_dims(tf.transpose(tf.pack(a.attentions_in_time), perm=[1, 2, 0]), -1)
            tf.image_summary("attention_{}".format(i), alignments,
                             collections=summary_collections, max_images=batch_size)

    return outputs, states

def decode_step(prev_output, prev_state, attention_objects,
                rnn_cell, maxout_size):
    """This function implements equations in section A.2.2 of the
    Bahdanau et al. (2015) paper, on pages 13 and 14.

    Arguments:
        prev_output: Previous decoded output (denoted by y_i-1)
        prev_state: Previous state (denoted by s_i-1)
        attention_objects: Objects that do attention
        rnn_cell: The RNN cell to use (should be GRU)
        maxout_size: The size of the maxout hidden layer (denoted by l)

    Returns:
        Tuple of the new output and state
    """
    ## compute c_i:
    contexts = [a.attention(prev_state) for a in attention_objects]

    # TODO dropouts??

    ## compute t_i:
    output = maxout([prev_state, prev_output] + contexts, maxout_size)

    ## compute s_i based on y_i-1, c_i and s_i-1
    _, state = rnn_cell(tf.concat(1, [prev_output] + contexts), prev_state)

    return output, state

class Attention(object):
    #pylint: disable=unused-argument,too-many-instance-attributes
    # For maintaining the same API as in CoverageAttention
    def __init__(self, attention_states, scope, dropout_placeholder,
                 input_weights=None, max_fertility=None):
        self.scope = scope
        self.attentions_in_time = []
        self.attention_states = tf.nn.dropout(attention_states,
                                              dropout_placeholder)
        self.input_weights = input_weights

        with tf.variable_scope(scope):
            self.attn_length = attention_states.get_shape()[1].value
            self.attn_size = attention_states.get_shape()[2].value

            # To calculate W1 * h_t we use a 1-by-1 convolution, need to reshape
            # before.
            self.att_states_reshaped = tf.reshape(
                self.attention_states,
                [-1, self.attn_length, 1, self.attn_size])

            self.attention_vec_size = self.attn_size    # Size of query vectors
                                                        # for attention.

            k = tf.get_variable(
                "AttnW",
                [1, 1, self.attn_size, self.attention_vec_size])

            self.hidden_features = tf.nn.conv2d(self.att_states_reshaped, k,
                                                [1, 1, 1, 1], "SAME")

            #pylint: disable=invalid-name
            # see comments on disabling invalid names below
            self.v = tf.get_variable("AttnV", [self.attention_vec_size])

    def attention(self, query_state):
        """Put attention masks on att_states_reshaped
           using hidden_features and query.
        """

        with tf.variable_scope(self.scope+"/Attention"):
            y = linear(query_state, self.attention_vec_size)
            y = tf.reshape(y, [-1, 1, 1, self.attention_vec_size])

            #pylint: disable=invalid-name
            # code copied from tensorflow. Suggestion: rename the variables
            # according to the Bahdanau paper
            s = self.get_logits(y)

            if self.input_weights is None:
                a = tf.nn.softmax(s)
            else:
                a_all = tf.nn.softmax(s) * self.input_weights
                norm = tf.reduce_sum(a_all, 1, keep_dims=True) + 1e-8
                a = a_all / norm

            self.attentions_in_time.append(a)

            # Now calculate the attention-weighted vector d.
            d = tf.reduce_sum(tf.reshape(a, [-1, self.attn_length, 1, 1])
                              * self.att_states_reshaped, [1, 2])

            return tf.reshape(d, [-1, self.attn_size])

    def get_logits(self, y):
        # Attention mask is a softmax of v^T * tanh(...).
        return tf.reduce_sum(self.v * tf.tanh(self.hidden_features + y), [2, 3])

class CoverageAttention(Attention):

    # pylint: disable=too-many-arguments
    # Great objects require great number of parameters
    def __init__(self, attention_states, scope, dropout_placeholder,
                 input_weights=None, max_fertility=5):

        super(CoverageAttention, self).__init__(attention_states, scope,
                                                dropout_placeholder,
                                                input_weights=input_weights,
                                                max_fertility=max_fertility)

        self.coverage_weights = tf.get_variable("coverage_matrix",
                                                [1, 1, 1, self.attn_size])
        self.fertility_weights = tf.get_variable("fertility_matrix",
                                                 [1, 1, self.attn_size])
        self.max_fertility = max_fertility

        self.fertility = 1e-8 + self.max_fertility * tf.sigmoid(
            tf.reduce_sum(self.fertility_weights * self.attention_states, [2]))

    def get_logits(self, y):
        coverage = sum(
            self.attentions_in_time) / self.fertility * self.input_weights

        logits = tf.reduce_sum(
            self.v * tf.tanh(
                self.hidden_features + y + self.coverage_weights * tf.reshape(
                    coverage, [-1, self.attn_length, 1, 1])),
            [2, 3])

        return logits
<|endoftext|>


<commit_before>from collections import OrderedDict
import scipy.optimize as scimin
from scipy.integrate import odeint
import matplotlib.pyplot as plt
import numpy
import lmfit

param_order = [
            'Tau_r0', 'a_FDR', 'Tau_FDR', 'Tau_f', 'p0bar',
            'a_i', 'Tau_i', 'a_D', 'Tau_D', 'Tau_r', 'p0'
        ]

dynamics_reverse_order={'0':'Dep','1':'Fac','2':'UR','3':'SMR','4':'DSR'}
ode_variables_reverse_order={'0':'n','1':'p','2':'Tau_r','3':'p0','4':'D'}
dynamics_order={'Dep':0,'Fac':1,'UR':2,'SMR':3,'DSR':4}
ode_variables_order={'n':0,'p':1,'Tau_r':2,'p0':3,'D':4}
dynamics_types = ['Dep', 'Fac', 'UR', 'SMR', 'DSR']

def f(y, t, all_dict,gating):
    """ Calculate derivatives of state variables between spikes.

    Parameters
    ----------
    y : array
        state variables of synapses
    t : float
        time
    all_dict : dictionary
        parameters necessary to calculate derivatives of states
    gating: dictionary
        specify state variables necessary to be calculated.

    Returns
    -------
    derivs : array
       Derivatives of state variables.
    """
    # unpack params
    #a_n, Tau_r0, a_FDR, Tau_FDR, a_f, Tau_f, p0bar, a_i, Tau_i, a_D, Tau_D, blah = params
    n, p, Tau_r, p0, D = y      # unpack current values of y
    #print y
    #print n
    #print all_dict,gating
    derivs =numpy.zeros(len(y))
    #print Tau_r0, Tau_r
    initial_y=n
   # print gating
    if gating['Dep']==1:
        if gating['UR']==1:
            derivs[0] = (1-n) /Tau_r
        else:
            derivs[0] = (1-n) / all_dict['Tau_r']

    if gating['Fac']==1:
        if gating['SMR']==1:
            derivs[1] = (p0-p) / all_dict['Tau_f']
        else:
            derivs[1] = (all_dict['p0']-p) / all_dict['Tau_f']

    if gating['UR']==1:
        derivs[2] = (all_dict['Tau_r0']-Tau_r) /all_dict['Tau_FDR']

    if gating['SMR']==1:
        derivs[3] = (all_dict['p0bar']-p0) /all_dict['Tau_i']

    if gating['DSR']==1:
        derivs[4] = (1-D) /all_dict['Tau_D']
    #print initial_y
    return derivs

def feval(spikes, length_array, dynamics,ode_variables, Tau_r0, a_FDR, Tau_FDR, p0, Tau_f, p0bar, a_i, Tau_i, a_D, Tau_D, Tau_r,):
    """Evaluate release model, returning an array of state parameters
    for each timepoint in spike_times.

    Parameters
    ----------
    spikes : list
        Times of presyanptic spikes
    length_array : list
        Length of each sweep, which is necessary to reconstruct the fitting results into the origial form of the experiments
    dynamics : array
        Five flags to specify which state variables are necessary to be estimated; each flag determines if the corresponding state needs to be calculated.
    ode_variables:array
        Order of state variables, which are necessary to pass them as a vector (without keywords)
    Tau_r0, a_FDR, Tau_FDR, p0, Tau_f, p0bar, a_i, Tau_i, a_D, Tau_D, Tau_r: floats
        Values of parameters.
    Returns
    -------
    output : array
        PSPs estimated for spike times.
    """

    #print spikes
    length=length_array
    spikes_cont=[]
    st=int(0)
    for xin in length:
        spikes_cont.append(spikes[st:st+int(xin)])
        #print datax[st:st+xin]
        st=st+int(xin)
    #print spikes_cont

    param_dict={}

    #param_dict['a_n']=a_n
    param_dict['Tau_r0']=Tau_r0
    param_dict['a_FDR']=a_FDR
    param_dict['Tau_FDR']=Tau_FDR
    #param_dict['a_f']=a_f
    param_dict['Tau_f']=Tau_f
    param_dict['p0bar']=p0bar
    param_dict['a_i']=a_i
    param_dict['Tau_i']=Tau_i
    param_dict['a_D']=a_D
    param_dict['p0']=p0
    param_dict['Tau_D']=Tau_D
    param_dict['Tau_r']=Tau_r
    #print Tau_r

    gating={}
    for di, dn in enumerate(dynamics):
        gating[dynamics_reverse_order[str(di)]]=dn
    #print gating
    ode_vs={}
    for di, dn in enumerate(ode_variables):
        ode_vs[ode_variables_reverse_order[str(di)]]=dn

    dt=1.0
    return_v=[]

    for ex in spikes_cont:
        spike_times=numpy.array(ex)
        y=numpy.ones(5)
        y[1]=param_dict['p0']
        #p0_initial=y[1]
        if ode_vs['Tau_r']==1:
            y[2]=param_dict['Tau_r0'] # initial value
        if ode_vs['p0']==1:
            y[3]=param_dict['p0bar']  # intial value
            y[1]=y[3]

        nspikes = len(spike_times)
        #print nspikes
        output = numpy.zeros((nspikes, 2 + len(y)))
        yw = numpy.zeros((nspikes, 1))
        output[0, 0] = 0
        yw[0, 0] = 1 # p should be renormalized to the p0.
        output[0, 2:] = y

        # Loop over spikes and evaluation timepoints
        # Each spike causes an instantaneous change in state parameters,
        # and then we integrate the ode until the next spike.
        for i in range(0, nspikes):
            if i!=0:
                # Integrate until next spike

                # Instantaneous changes in state induced by spike
                time_pts = numpy.arange(spike_times[i-1], spike_times[i], dt) + dt
                #print time_pts
                psoln = odeint(f, y, time_pts, args=(param_dict,gating))
                # Last time point becomes the initial state for the next spike
                y = psoln[-1]
                # Record state for this timepoint
                output[i, 0] = time_pts[-1]
                output[i, 2:] = psoln[-1]

            yp = y[:]
            y01 = y[0]*y[1]

            if gating['Dep']==1:
                #y01 *=
                #if gating['Fac']==1:
                #    y01 *= y[1]
                yp[0] -=y01

            if gating['Fac']==1:
                if gating['SMR']==1:
                    yp[1] +=yp[3]*(1-y[1])
                else:
                    yp[1] +=param_dict['p0']*(1-y[1])

            if gating['UR']==1:
                yp[2] -= param_dict['a_FDR'] * y[2]

            if gating['SMR']==1:
                yp[3] -=param_dict['a_i'] * y[3]

            if gating['DSR']==1:
                yp[4] -=param_dict['a_D'] * y01 * y[4]

            y = yp[:]
            if i==0:
                initial_w=y[0]*y[1]*y[4]

            yw[i,0]=y[0]*y[1]*y[4]/initial_w
            #print y[1],yw[1],p0_initial
            #print y

            #Generate n*p*D column
        output[:, 1] = yw[:,0]
        return_v.extend(output[:, 1])
        #print ex,i,len(output[:, 1])
    #print numpy.array(return_v)
    return numpy.array(return_v)

class ReleaseModel(object):
    """Presynaptic release model based on Hennig et al. 2013
        The input should be a list of tuples [(x, y), ...], where each tuple
        contains the spike times (x) and PSG amplitudes (y) to fit against.

        Returns amplitudes predicted by model and fit results provided by lmfit
        The shape of amplitudes is the same as that of spike_set (the inputs).

    Jung Hoon Lee <jungl@alleninstitute.org>

    """

    ode_variables={'n':1,'p':1,'Tau_r':1,'p0':1,'D':1}
    def __init__(self,Dynamics,initial_params,bound_params):
        # Dep: depression
        # Fac: facilitation
        # UR: Use-dependent replentishment
        # SMR: Slow modulation of release
        # DSR: Receptor desensitization

        self.Dynamics=Dynamics
        self.dict_params =initial_params

        self.order =param_order

        self.dict_bounds=bound_params

    def run_fit(self, spike_sets):
        """Fit the model against data.

        The input should be a list of tuples [(x, y), ...], where each tuple
        contains the spike times (x) and PSG amplitudes (y) to fit against.

        Returns a dictionary of optimal model parameters.
        """
        #print spike_sets
        self.data_x=[]
        self.data_y=[]
        self.data_e=[]
        lengths=[]
        for datax, datay in spike_sets:
            self.data_x.extend(datax)
            self.data_y.extend(datay)
            #self.data_e.extend(dataz)
            lengths.append(len(datax))
        print self.Dynamics

        params=lmfit.Parameters()
        self.Sel_gatings=[]
        if self.Dynamics['Dep']==1:
            if self.Dynamics['Fac']==0:
                self.Sel_gatings.append('p0')
            if self.Dynamics['UR']==1:
                self.Sel_gatings.append('Tau_r0')
                self.ode_variables['Tau_r']=1
            else:
                self.ode_variables['n']=1
                self.Sel_gatings.append('Tau_r')

        else:
            self.ode_variables['n']=0 #it must not be used since all stp is dependent on the depletion model.

        if self.Dynamics['Fac']==1:
            #self.Sel_gatings.append('a_f')
            self.Sel_gatings.append('Tau_f')
            self.ode_variables['p']=1
            if self.Dynamics['SMR']==1:
                self.Sel_gatings.append('p0bar')
                self.ode_variables['p0']=1
            else:
                self.Sel_gatings.append('p0')
        else:
            self.ode_variables['p']=0

        if self.Dynamics['UR']==1:
            self.Sel_gatings.append('a_FDR')
            self.Sel_gatings.append('Tau_FDR')

            if self.Dynamics['Dep']==0:
                self.Sel_gatings.append('Tau_r0')
                self.ode_variables['Tau_r']=1
        else:
            self.ode_variables['Tau_r']=0

        if self.Dynamics['SMR']==1:
            self.Sel_gatings.append('a_i')
            self.Sel_gatings.append('Tau_i')
            if self.Dynamics['Fac']==0:
                self.Sel_gatings.append('p0bar')
                self.ode_variables['p0']=1
        else:
            self.ode_variables['p0']=0

        if self.Dynamics['DSR']==1:
            self.Sel_gatings.append('a_D')
            self.Sel_gatings.append('Tau_D')
            self.ode_variables['D']=1
        else:
            self.ode_variables['D']=0
        dynamics_vec=numpy.zeros(5)
        ode_variables_vec=numpy.zeros(5)
        for dn in dynamics_order:
            dynamics_vec[dynamics_order[dn]]=self.Dynamics[dn]

        for dn in ode_variables_order:
            ode_variables_vec[ode_variables_order[dn]]=self.ode_variables[dn]
        self.freeparam=0
        fitmodel=lmfit.Model(feval,independent_vars=['spikes','length_array','dynamics', 'ode_variables'])
        for pm in self.order:
            #print 'pm0',pm
            if pm in self.Sel_gatings:
                print 'variable pm',pm
                self.freeparam=self.freeparam+1
                fitmodel.set_param_hint(pm,vary=True,value=self.dict_params[pm],min=self.dict_bounds[pm][0],max=self.dict_bounds[pm][1])
            else:
                print 'fixed',pm
                fitmodel.set_param_hint(pm,vary=False,value=self.dict_params[pm],min=self.dict_bounds[pm][0],max=self.dict_bounds[pm][1])
        pars=fitmodel.make_params()
        print pars
        result = fitmodel.fit(numpy.array(self.data_y),spikes=numpy.array(self.data_x),length_array=lengths,dynamics=dynamics_vec,ode_variables=ode_variables_vec)
        print(result.fit_report())
        #ci=lmfit.conf_interval(fitmodel,result)
        #lmfit.printfunc.report_ci(ci)
        self.model_y=result.eval()
        model_ys=[]
        ct=0
        for xi,xin in enumerate(lengths):
            model_ys.append(self.model_y[ct:ct+xin])
            ct=ct+xin

        return model_ys,result

    def goodness_of_fit(self):
        """ Calculate derivatives of state variables between spikes.

            Returns
            -------
            derivs : array
            r^2, adj. r^2
       """
        data_mean=numpy.mean(self.data_y)
        ss_t=0
        ss_res=0
        p=float(self.freeparam)
        n=float(len(self.data_y))
        for xi,xin in enumerate(self.data_y):
            ss_t=ss_t+(xin-data_mean)**2
            ss_res=ss_res+(xin-self.model_y[xi])**2
        r_square=1-float(ss_res)/float(ss_t)
        adj_r_square=1-(1-r_square)*(n-1)/(n-p-1)
        R_s=numpy.array([r_square,adj_r_square])
        return R_s

<commit_msg>more fixes<commit_after>from collections import OrderedDict
import scipy.optimize as scimin
from scipy.integrate import odeint
import numpy
import lmfit

param_order = [
            'Tau_r0', 'a_FDR', 'Tau_FDR', 'Tau_f', 'p0bar',
            'a_i', 'Tau_i', 'a_D', 'Tau_D', 'Tau_r', 'p0'
        ]

dynamics_reverse_order={'0':'Dep','1':'Fac','2':'UR','3':'SMR','4':'DSR'}
ode_variables_reverse_order={'0':'n','1':'p','2':'Tau_r','3':'p0','4':'D'}
dynamics_order={'Dep':0,'Fac':1,'UR':2,'SMR':3,'DSR':4}
ode_variables_order={'n':0,'p':1,'Tau_r':2,'p0':3,'D':4}
dynamics_types = ['Dep', 'Fac', 'UR', 'SMR', 'DSR']

def f(y, t, all_dict,gating):
    """ Calculate derivatives of state variables between spikes.

    Parameters
    ----------
    y : array
        state variables of synapses
    t : float
        time
    all_dict : dictionary
        parameters necessary to calculate derivatives of states
    gating: dictionary
        specify state variables necessary to be calculated.

    Returns
    -------
    derivs : array
       Derivatives of state variables.
    """
    # unpack params
    #a_n, Tau_r0, a_FDR, Tau_FDR, a_f, Tau_f, p0bar, a_i, Tau_i, a_D, Tau_D, blah = params
    n, p, Tau_r, p0, D = y      # unpack current values of y
    #print y
    #print n
    #print all_dict,gating
    derivs =numpy.zeros(len(y))
    #print Tau_r0, Tau_r
    initial_y=n
   # print gating
    if gating['Dep']==1:
        if gating['UR']==1:
            derivs[0] = (1-n) /Tau_r
        else:
            derivs[0] = (1-n) / all_dict['Tau_r']

    if gating['Fac']==1:
        if gating['SMR']==1:
            derivs[1] = (p0-p) / all_dict['Tau_f']
        else:
            derivs[1] = (all_dict['p0']-p) / all_dict['Tau_f']

    if gating['UR']==1:
        derivs[2] = (all_dict['Tau_r0']-Tau_r) /all_dict['Tau_FDR']

    if gating['SMR']==1:
        derivs[3] = (all_dict['p0bar']-p0) /all_dict['Tau_i']

    if gating['DSR']==1:
        derivs[4] = (1-D) /all_dict['Tau_D']
    #print initial_y
    return derivs

def feval(spikes, length_array, dynamics,ode_variables, Tau_r0, a_FDR, Tau_FDR, p0, Tau_f, p0bar, a_i, Tau_i, a_D, Tau_D, Tau_r,):
    """Evaluate release model, returning an array of state parameters
    for each timepoint in spike_times.

    Parameters
    ----------
    spikes : list
        Times of presyanptic spikes
    length_array : list
        Length of each sweep, which is necessary to reconstruct the fitting results into the origial form of the experiments
    dynamics : array
        Five flags to specify which state variables are necessary to be estimated; each flag determines if the corresponding state needs to be calculated.
    ode_variables:array
        Order of state variables, which are necessary to pass them as a vector (without keywords)
    Tau_r0, a_FDR, Tau_FDR, p0, Tau_f, p0bar, a_i, Tau_i, a_D, Tau_D, Tau_r: floats
        Values of parameters.
    Returns
    -------
    output : array
        PSPs estimated for spike times.
    """

    #print spikes
    length=length_array
    spikes_cont=[]
    st=int(0)
    for xin in length:
        spikes_cont.append(spikes[st:st+int(xin)])
        #print datax[st:st+xin]
        st=st+int(xin)
    #print spikes_cont

    param_dict={}

    #param_dict['a_n']=a_n
    param_dict['Tau_r0']=Tau_r0
    param_dict['a_FDR']=a_FDR
    param_dict['Tau_FDR']=Tau_FDR
    #param_dict['a_f']=a_f
    param_dict['Tau_f']=Tau_f
    param_dict['p0bar']=p0bar
    param_dict['a_i']=a_i
    param_dict['Tau_i']=Tau_i
    param_dict['a_D']=a_D
    param_dict['p0']=p0
    param_dict['Tau_D']=Tau_D
    param_dict['Tau_r']=Tau_r
    #print Tau_r

    gating={}
    for di, dn in enumerate(dynamics):
        gating[dynamics_reverse_order[str(di)]]=dn
    #print gating
    ode_vs={}
    for di, dn in enumerate(ode_variables):
        ode_vs[ode_variables_reverse_order[str(di)]]=dn

    dt=1.0
    return_v=[]

    for ex in spikes_cont:
        spike_times=numpy.array(ex)
        y=numpy.ones(5)
        y[1]=param_dict['p0']
        #p0_initial=y[1]
        if ode_vs['Tau_r']==1:
            y[2]=param_dict['Tau_r0'] # initial value
        if ode_vs['p0']==1:
            y[3]=param_dict['p0bar']  # intial value
            y[1]=y[3]

        nspikes = len(spike_times)
        #print nspikes
        output = numpy.zeros((nspikes, 2 + len(y)))
        yw = numpy.zeros((nspikes, 1))
        output[0, 0] = 0
        yw[0, 0] = 1 # p should be renormalized to the p0.
        output[0, 2:] = y

        # Loop over spikes and evaluation timepoints
        # Each spike causes an instantaneous change in state parameters,
        # and then we integrate the ode until the next spike.
        for i in range(0, nspikes):
            if i!=0:

                # Instantaneous changes in state induced by spike
                time_pts = numpy.arange(spike_times[i-1], spike_times[i], dt) + dt
                # Integrate until next spike
                psoln = odeint(f, y, time_pts, args=(param_dict,gating))
                # Last time point becomes the initial state for the next spike
                y = psoln[-1]
                # Record state for this timepoint
                output[i, 0] = time_pts[-1]
                output[i, 2:] = psoln[-1]

            yp = y[:]
            y01 = y[0]*y[1]

            if gating['Dep']==1:
                #y01 *=
                #if gating['Fac']==1:
                #    y01 *= y[1]
                yp[0] -=y01

            if gating['Fac']==1:
                if gating['SMR']==1:
                    yp[1] +=yp[3]*(1-y[1])
                else:
                    yp[1] +=param_dict['p0']*(1-y[1])

            if gating['UR']==1:
                yp[2] -= param_dict['a_FDR'] * y[2]

            if gating['SMR']==1:
                yp[3] -=param_dict['a_i'] * y[3]

            if gating['DSR']==1:
                yp[4] -=param_dict['a_D'] * y01 * y[4]

            y = yp[:]
            if i==0:
                initial_w=y[0]*y[1]*y[4]

            yw[i,0]=y[0]*y[1]*y[4]/initial_w
            #print y[1],yw[1],p0_initial
            #print y

            #Generate n*p*D column
        output[:, 1] = yw[:,0]
        return_v.extend(output[:, 1])
        #print ex,i,len(output[:, 1])
    #print numpy.array(return_v)
    return numpy.array(return_v)

class ReleaseModel(object):
    """Presynaptic release model based on Hennig et al. 2013
        The input should be a list of tuples [(x, y), ...], where each tuple
        contains the spike times (x) and PSG amplitudes (y) to fit against.

        Returns amplitudes predicted by model and fit results provided by lmfit
        The shape of amplitudes is the same as that of spike_set (the inputs).

    Jung Hoon Lee <jungl@alleninstitute.org>

    """

    def __init__(self,Dynamics,initial_params,bound_params):
        # Dep: depression
        # Fac: facilitation
        # UR: Use-dependent replentishment
        # SMR: Slow modulation of release
        # DSR: Receptor desensitization
        self.ode_variable=={'n':1,'p':1,'Tau_r':1,'p0':1,'D':1}
        self.Dynamics=Dynamics
        self.dict_params =initial_params

        self.order =param_order

        self.dict_bounds=bound_params

    def run_fit(self, spike_sets):
        """Fit the model against data.

        The input should be a list of tuples [(x, y), ...], where each tuple
        contains the spike times (x) and PSG amplitudes (y) to fit against.

        Returns a dictionary of optimal model parameters.
        """
        #print spike_sets
        self.data_x=[]
        self.data_y=[]
        self.data_e=[]
        lengths=[]
        for datax, datay in spike_sets:
            self.data_x.extend(datax)
            self.data_y.extend(datay)
            #self.data_e.extend(dataz)
            lengths.append(len(datax))
        print self.Dynamics

        params=lmfit.Parameters()
        self.Sel_gatings=[]
        if self.Dynamics['Dep']==1:
            if self.Dynamics['Fac']==0:
                self.Sel_gatings.append('p0')
            if self.Dynamics['UR']==1:
                self.Sel_gatings.append('Tau_r0')
                self.ode_variables['Tau_r']=1
            else:
                self.ode_variables['n']=1
                self.Sel_gatings.append('Tau_r')

        else:
            self.ode_variables['n']=0 #it must not be used since all stp is dependent on the depletion model.

        if self.Dynamics['Fac']==1:
            #self.Sel_gatings.append('a_f')
            self.Sel_gatings.append('Tau_f')
            self.ode_variables['p']=1
            if self.Dynamics['SMR']==1:
                self.Sel_gatings.append('p0bar')
                self.ode_variables['p0']=1
            else:
                self.Sel_gatings.append('p0')
        else:
            self.ode_variables['p']=0

        if self.Dynamics['UR']==1:
            self.Sel_gatings.append('a_FDR')
            self.Sel_gatings.append('Tau_FDR')

            if self.Dynamics['Dep']==0:
                self.Sel_gatings.append('Tau_r0')
                self.ode_variables['Tau_r']=1
        else:
            self.ode_variables['Tau_r']=0

        if self.Dynamics['SMR']==1:
            self.Sel_gatings.append('a_i')
            self.Sel_gatings.append('Tau_i')
            if self.Dynamics['Fac']==0:
                self.Sel_gatings.append('p0bar')
                self.ode_variables['p0']=1
        else:
            self.ode_variables['p0']=0

        if self.Dynamics['DSR']==1:
            self.Sel_gatings.append('a_D')
            self.Sel_gatings.append('Tau_D')
            self.ode_variables['D']=1
        else:
            self.ode_variables['D']=0
        dynamics_vec=numpy.zeros(5)
        ode_variables_vec=numpy.zeros(5)
        for dn in dynamics_order:
            dynamics_vec[dynamics_order[dn]]=self.Dynamics[dn]

        for dn in ode_variables_order:
            ode_variables_vec[ode_variables_order[dn]]=self.ode_variables[dn]
        self.freeparam=0
        fitmodel=lmfit.Model(feval,independent_vars=['spikes','length_array','dynamics', 'ode_variables'])
        for pm in self.order:
            #print 'pm0',pm
            if pm in self.Sel_gatings:
                print 'variable pm',pm
                self.freeparam=self.freeparam+1
                fitmodel.set_param_hint(pm,vary=True,value=self.dict_params[pm],min=self.dict_bounds[pm][0],max=self.dict_bounds[pm][1])
            else:
                print 'fixed',pm
                fitmodel.set_param_hint(pm,vary=False,value=self.dict_params[pm],min=self.dict_bounds[pm][0],max=self.dict_bounds[pm][1])
        pars=fitmodel.make_params()
        print pars
        result = fitmodel.fit(numpy.array(self.data_y),spikes=numpy.array(self.data_x),length_array=lengths,dynamics=dynamics_vec,ode_variables=ode_variables_vec)
        print(result.fit_report())
        #ci=lmfit.conf_interval(fitmodel,result)
        #lmfit.printfunc.report_ci(ci)
        self.model_y=result.eval()
        model_ys=[]
        ct=0
        for xi,xin in enumerate(lengths):
            model_ys.append(self.model_y[ct:ct+xin])
            ct=ct+xin

        return model_ys,result

    def goodness_of_fit(self):
        """ Calculate derivatives of state variables between spikes.

            Returns
            -------
            derivs : array
            r^2, adj. r^2
       """
        data_mean=numpy.mean(self.data_y)
        ss_t=0
        ss_res=0
        p=float(self.freeparam)
        n=float(len(self.data_y))
        for xi,xin in enumerate(self.data_y):
            ss_t=ss_t+(xin-data_mean)**2
            ss_res=ss_res+(xin-self.model_y[xi])**2
        r_square=1-float(ss_res)/float(ss_t)
        adj_r_square=1-(1-r_square)*(n-1)/(n-p-1)
        R_s=numpy.array([r_square,adj_r_square])
        return R_s

<|endoftext|>