<commit_before># -*- coding: utf-8 -*-
# Generated by Django 1.11.23 on 2020-01-04 21:57
from __future__ import unicode_literals

import django.contrib.postgres.fields
import django.contrib.postgres.fields.hstore
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
import django.db.models.deletion

class Migration(migrations.Migration):

    dependencies = [
        ('hs_core', '0047_auto_20200512_1509'),
        ('hs_composite_resource', '0001_initial'),
        ('hs_file_types', '0010_auto_20181209_0255'),
    ]

    operations = [
        migrations.CreateModel(
            name='ModelProgramFileMetaData',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('extra_metadata', django.contrib.postgres.fields.hstore.HStoreField(default={})),
                ('keywords', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=100, null=True), default=[], size=None)),
                ('is_dirty', models.BooleanField(default=False)),
                ('version', models.CharField(blank=True, help_text=b'The software version or build number of the model', max_length=255, null=True, verbose_name=b'Version')),
                ('programming_languages', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=100, null=True), default=list, help_text=b'The programming language(s) that the model is written in', size=None)),
                ('operating_systems', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=100, null=True), default=list, help_text=b'Compatible operating systems to setup and run the model', size=None)),
                ('release_date', models.DateField(blank=True, help_text=b'The date that this version of the model was released', null=True, verbose_name=b'Release Date')),
                ('website', models.URLField(blank=True, help_text=b'A URL to the website maintained by the model developers', max_length=255, null=True, verbose_name=b'Website')),
                ('code_repository', models.URLField(blank=True, help_text=b'A URL to the source code repository (e.g. git, mercurial, svn)', max_length=255, null=True, verbose_name=b'Software Repository')),
            ],
            options={
                'abstract': False,
            },
        ),
        migrations.CreateModel(
            name='ModelProgramLogicalFile',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('dataset_name', models.CharField(blank=True, max_length=255, null=True)),
                ('extra_data', django.contrib.postgres.fields.hstore.HStoreField(default={})),
                ('model_program_type', models.CharField(default=b'Unknown Model Program', max_length=255)),
                ('mi_schema_json', django.contrib.postgres.fields.jsonb.JSONField(default=dict)),
                ('folder', models.CharField(blank=True, max_length=4096, null=True)),
                ('metadata', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='logical_file', to='hs_file_types.ModelProgramFileMetaData')),
                ('resource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='hs_composite_resource.CompositeResource')),
            ],
            options={
                'abstract': False,
            },
        ),
        migrations.CreateModel(
            name='ModelProgramResourceFileType',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('file_type', models.PositiveSmallIntegerField(choices=[(1, b'Release Notes'), (2, b'Documentation'), (3, b'Software'), (4, b'Computational Engine')])),
                ('mp_metadata', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='mp_file_types', to='hs_file_types.ModelProgramFileMetaData')),
                ('res_file', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='hs_core.ResourceFile')),
            ],
        ),
    ]
<commit_msg>[#3519] updating migration reference<commit_after># -*- coding: utf-8 -*-
# Generated by Django 1.11.23 on 2020-01-04 21:57
from __future__ import unicode_literals

import django.contrib.postgres.fields
import django.contrib.postgres.fields.hstore
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
import django.db.models.deletion

class Migration(migrations.Migration):

    dependencies = [
        ('hs_core', '0050_auto_20200611_1912'),
        ('hs_composite_resource', '0001_initial'),
        ('hs_file_types', '0010_auto_20181209_0255'),
    ]

    operations = [
        migrations.CreateModel(
            name='ModelProgramFileMetaData',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('extra_metadata', django.contrib.postgres.fields.hstore.HStoreField(default={})),
                ('keywords', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=100, null=True), default=[], size=None)),
                ('is_dirty', models.BooleanField(default=False)),
                ('version', models.CharField(blank=True, help_text=b'The software version or build number of the model', max_length=255, null=True, verbose_name=b'Version')),
                ('programming_languages', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=100, null=True), default=list, help_text=b'The programming language(s) that the model is written in', size=None)),
                ('operating_systems', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=100, null=True), default=list, help_text=b'Compatible operating systems to setup and run the model', size=None)),
                ('release_date', models.DateField(blank=True, help_text=b'The date that this version of the model was released', null=True, verbose_name=b'Release Date')),
                ('website', models.URLField(blank=True, help_text=b'A URL to the website maintained by the model developers', max_length=255, null=True, verbose_name=b'Website')),
                ('code_repository', models.URLField(blank=True, help_text=b'A URL to the source code repository (e.g. git, mercurial, svn)', max_length=255, null=True, verbose_name=b'Software Repository')),
            ],
            options={
                'abstract': False,
            },
        ),
        migrations.CreateModel(
            name='ModelProgramLogicalFile',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('dataset_name', models.CharField(blank=True, max_length=255, null=True)),
                ('extra_data', django.contrib.postgres.fields.hstore.HStoreField(default={})),
                ('model_program_type', models.CharField(default=b'Unknown Model Program', max_length=255)),
                ('mi_schema_json', django.contrib.postgres.fields.jsonb.JSONField(default=dict)),
                ('folder', models.CharField(blank=True, max_length=4096, null=True)),
                ('metadata', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='logical_file', to='hs_file_types.ModelProgramFileMetaData')),
                ('resource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='hs_composite_resource.CompositeResource')),
            ],
            options={
                'abstract': False,
            },
        ),
        migrations.CreateModel(
            name='ModelProgramResourceFileType',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('file_type', models.PositiveSmallIntegerField(choices=[(1, b'Release Notes'), (2, b'Documentation'), (3, b'Software'), (4, b'Computational Engine')])),
                ('mp_metadata', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='mp_file_types', to='hs_file_types.ModelProgramFileMetaData')),
                ('res_file', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='hs_core.ResourceFile')),
            ],
        ),
    ]
<|endoftext|>


<commit_before>import torch
import torch.nn as nn
from torch import optim
import torch.nn.functional as F

class MLPNet(nn.Module):
    '''
    Class for generating arbitrary sized feedforward
    neural network, with ReLU activations
    '''

    def __init__(self,
                 in_dim,
                 hid_dim,
                 out_dim,
                 optim=optim.Adam,
                 loss_fn=F.smooth_l1_loss,
                 clamp_grad=False):
        '''
        in_dim: dimension of the inputs
        hid_dim: list containing dimensions of the hidden layers
        out_dim: dimension of the ouputs
        optim: optimizer
        loss_fn: measure of error between model predictions and correct outputs
        clamp_grad: whether to clamp the gradient to + / - 1
        example:
        net = MLPNet(1000, [512, 256, 128], 10, optim.Adam, nn.SmoothL1Loss)
        '''
        super(MLPNet, self).__init__()
        self.in_dim = in_dim
        self.out_dim = out_dim
        self.hid_layers = []
        self.batch_norms = []
        for i, layer in enumerate(hid_dim):
            if i == 0:
                in_D = in_dim
            else:
                in_D = hid_dim[i - 1]
            out_D = hid_dim[i]
            l = nn.Linear(in_D, out_D)
            setattr(self, 'linear_' + str(i), l)
            self.hid_layers.append(l)
        if len(hid_dim) > 0:
            self.out_layer = nn.Linear(hid_dim[-1], out_dim)
        else:
            self.out_layer = nn.Linear(in_dim, out_dim)
        self.num_hid_layers = len(self.hid_layers)
        self.optim = optim(self.parameters())
        self.loss_fn = loss_fn
        self.clamp_grad = clamp_grad
        self.init_params()

    def forward(self, x):
        ''' The feedforward step '''
        for i in range(self.num_hid_layers):
            x = F.relu((self.hid_layers[i](x)))
        x = self.out_layer(x)
        return x

    def training_step(self, x, y):
        '''
        Takes a single training step: one forwards and one backwards pass
        '''
        # Sets model in training mode and zero the gradients
        self.train()
        self.optim.zero_grad()
        out = self(x)
        loss = self.loss_fn(out, y)
        loss.backward()
        if self.clamp_grad:
            for param in self.parameters():
                param.grad.data.clamp_(-1, 1)
        self.optim.step()
        return loss

    def eval(self, x):
        '''
        Completes one feedforward step, ensuring net is set to evaluation model
        returns: network output given input x
        '''
        self.eval()
        return self(x)

    def init_params(self):
        '''
        Initializes all of the model's parameters using uniform initialization.
        Note: Ideally it should be xavier initialization, but there appears
        to be unreproduceable behaviours in pyTorch.
        Sometimes the trainable params tests pass (see nn_test.py), other times
        they dont.
        Biases are all set to 0.01
        '''
        initrange = 0.2
        biasinit = 0.01
        lin_layers = self.hid_layers + list([self.out_layer])
        for layer in lin_layers:
            layer.weight.data.uniform_(-initrange, initrange)
            layer.bias.data.fill_(biasinit)

    def gather_trainable_params(self):
        '''
        Gathers parameters that should be trained into a list
        returns: copy of a list of fixed params
        '''
        return [param.clone() for param in self.parameters()]

    def gather_fixed_params(net):
        '''
        Gathers parameters that should be fixed into a list
        returns: copy of a list of fixed params
        '''
        return None
<commit_msg>MLPNet small fix (#10)<commit_after>import torch
import torch.nn as nn
from torch import optim
import torch.nn.functional as F

class MLPNet(nn.Module):
    '''
    Class for generating arbitrary sized feedforward
    neural network, with ReLU activations
    '''

    def __init__(self,
                 in_dim,
                 hid_dim,
                 out_dim,
                 optim=optim.Adam,
                 loss_fn=F.smooth_l1_loss,
                 clamp_grad=False):
        '''
        in_dim: dimension of the inputs
        hid_dim: list containing dimensions of the hidden layers
        out_dim: dimension of the ouputs
        optim: optimizer
        loss_fn: measure of error between model predictions and correct outputs
        clamp_grad: whether to clamp the gradient to + / - 1
        example:
        net = MLPNet(1000, [512, 256, 128], 10, optim.Adam, nn.SmoothL1Loss)
        '''
        super(MLPNet, self).__init__()
        self.in_dim = in_dim
        self.out_dim = out_dim
        self.hid_layers = []
        for i, layer in enumerate(hid_dim):
            if i == 0:
                in_D = in_dim
            else:
                in_D = hid_dim[i - 1]
            out_D = hid_dim[i]
            l = nn.Linear(in_D, out_D)
            setattr(self, 'linear_' + str(i), l)
            self.hid_layers.append(l)
        if len(hid_dim) > 0:
            self.out_layer = nn.Linear(hid_dim[-1], out_dim)
        else:
            self.out_layer = nn.Linear(in_dim, out_dim)
        self.num_hid_layers = len(self.hid_layers)
        self.optim = optim(self.parameters())
        self.loss_fn = loss_fn
        self.clamp_grad = clamp_grad
        self.init_params()

    def forward(self, x):
        ''' The feedforward step '''
        for i in range(self.num_hid_layers):
            x = F.relu((self.hid_layers[i](x)))
        x = self.out_layer(x)
        return x

    def training_step(self, x, y):
        '''
        Takes a single training step: one forwards and one backwards pass
        '''
        # Sets model in training mode and zero the gradients
        self.train()
        self.optim.zero_grad()
        out = self(x)
        loss = self.loss_fn(out, y)
        loss.backward()
        if self.clamp_grad:
            for param in self.parameters():
                param.grad.data.clamp_(-1, 1)
        self.optim.step()
        return loss

    def eval(self, x):
        '''
        Completes one feedforward step, ensuring net is set to evaluation model
        returns: network output given input x
        '''
        self.eval()
        return self(x)

    def init_params(self):
        '''
        Initializes all of the model's parameters using uniform initialization.
        Note: Ideally it should be xavier initialization, but there appears
        to be unreproduceable behaviours in pyTorch.
        Sometimes the trainable params tests pass (see nn_test.py), other times
        they dont.
        Biases are all set to 0.01
        '''
        initrange = 0.2
        biasinit = 0.01
        lin_layers = self.hid_layers + list([self.out_layer])
        for layer in lin_layers:
            layer.weight.data.uniform_(-initrange, initrange)
            layer.bias.data.fill_(biasinit)

    def gather_trainable_params(self):
        '''
        Gathers parameters that should be trained into a list
        returns: copy of a list of fixed params
        '''
        return [param.clone() for param in self.parameters()]

    def gather_fixed_params(self):
        '''
        Gathers parameters that should be fixed into a list
        returns: copy of a list of fixed params
        '''
        return None
<|endoftext|>


<commit_before># Copyright 2019 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Invocation-side implementation of gRPC Asyncio Python."""

import asyncio
import sys
from typing import Any, Iterable, List, Optional, Sequence

import grpc
from grpc import _common
from grpc import _compression
from grpc import _grpcio_metadata
from grpc._cython import cygrpc

from . import _base_call
from . import _base_channel
from ._call import StreamStreamCall
from ._call import StreamUnaryCall
from ._call import UnaryStreamCall
from ._call import UnaryUnaryCall
from ._interceptor import ClientInterceptor
from ._interceptor import InterceptedStreamStreamCall
from ._interceptor import InterceptedStreamUnaryCall
from ._interceptor import InterceptedUnaryStreamCall
from ._interceptor import InterceptedUnaryUnaryCall
from ._interceptor import StreamStreamClientInterceptor
from ._interceptor import StreamUnaryClientInterceptor
from ._interceptor import UnaryStreamClientInterceptor
from ._interceptor import UnaryUnaryClientInterceptor
from ._metadata import Metadata
from ._typing import ChannelArgumentType
from ._typing import DeserializingFunction
from ._typing import RequestIterableType
from ._typing import SerializingFunction
from ._utils import _timeout_to_deadline

_USER_AGENT = 'grpc-python-asyncio/{}'.format(_grpcio_metadata.__version__)

if sys.version_info[1] < 7:

    def _all_tasks() -> Iterable[asyncio.Task]:
        return asyncio.Task.all_tasks()
else:

    def _all_tasks() -> Iterable[asyncio.Task]:
        return asyncio.all_tasks()

def _augment_channel_arguments(base_options: ChannelArgumentType,
                               compression: Optional[grpc.Compression]):
    compression_channel_argument = _compression.create_channel_option(
        compression)
    user_agent_channel_argument = ((
        cygrpc.ChannelArgKey.primary_user_agent_string,
        _USER_AGENT,
    ),)
    return tuple(base_options
                ) + compression_channel_argument + user_agent_channel_argument

class _BaseMultiCallable:
    """Base class of all multi callable objects.

    Handles the initialization logic and stores common attributes.
    """
    _loop: asyncio.AbstractEventLoop
    _channel: cygrpc.AioChannel
    _method: bytes
    _request_serializer: SerializingFunction
    _response_deserializer: DeserializingFunction
    _interceptors: Optional[Sequence[ClientInterceptor]]
    _loop: asyncio.AbstractEventLoop

    # pylint: disable=too-many-arguments
    def __init__(
        self,
        channel: cygrpc.AioChannel,
        method: bytes,
        request_serializer: SerializingFunction,
        response_deserializer: DeserializingFunction,
        interceptors: Optional[Sequence[ClientInterceptor]],
        loop: asyncio.AbstractEventLoop,
    ) -> None:
        self._loop = loop
        self._channel = channel
        self._method = method
        self._request_serializer = request_serializer
        self._response_deserializer = response_deserializer
        self._interceptors = interceptors

    @staticmethod
    def _init_metadata(
            metadata: Optional[Metadata] = None,
            compression: Optional[grpc.Compression] = None) -> Metadata:
        """Based on the provided values for <metadata> or <compression> initialise the final
        metadata, as it should be used for the current call.
        """
        metadata = metadata or Metadata()
        if compression:
            metadata = Metadata(
                *_compression.augment_metadata(metadata, compression))
        return metadata

class UnaryUnaryMultiCallable(_BaseMultiCallable,
                              _base_channel.UnaryUnaryMultiCallable):

    def __call__(
        self,
        request: Any,
        *,
        timeout: Optional[float] = None,
        metadata: Optional[Metadata] = None,
        credentials: Optional[grpc.CallCredentials] = None,
        wait_for_ready: Optional[bool] = None,
        compression: Optional[grpc.Compression] = None
    ) -> _base_call.UnaryUnaryCall:

        metadata = self._init_metadata(metadata, compression)
        if not self._interceptors:
            call = UnaryUnaryCall(request, _timeout_to_deadline(timeout),
                                  metadata, credentials, wait_for_ready,
                                  self._channel, self._method,
                                  self._request_serializer,
                                  self._response_deserializer, self._loop)
        else:
            call = InterceptedUnaryUnaryCall(
                self._interceptors, request, timeout, metadata, credentials,
                wait_for_ready, self._channel, self._method,
                self._request_serializer, self._response_deserializer,
                self._loop)

        return call

class UnaryStreamMultiCallable(_BaseMultiCallable,
                               _base_channel.UnaryStreamMultiCallable):

    def __call__(
        self,
        request: Any,
        *,
        timeout: Optional[float] = None,
        metadata: Optional[Metadata] = None,
        credentials: Optional[grpc.CallCredentials] = None,
        wait_for_ready: Optional[bool] = None,
        compression: Optional[grpc.Compression] = None
    ) -> _base_call.UnaryStreamCall:

        metadata = self._init_metadata(metadata, compression)
        deadline = _timeout_to_deadline(timeout)

        if not self._interceptors:
            call = UnaryStreamCall(request, deadline, metadata, credentials,
                                   wait_for_ready, self._channel, self._method,
                                   self._request_serializer,
                                   self._response_deserializer, self._loop)
        else:
            call = InterceptedUnaryStreamCall(
                self._interceptors, request, deadline, metadata, credentials,
                wait_for_ready, self._channel, self._method,
                self._request_serializer, self._response_deserializer,
                self._loop)

        return call

class StreamUnaryMultiCallable(_BaseMultiCallable,
                               _base_channel.StreamUnaryMultiCallable):

    def __call__(
        self,
        request_iterator: Optional[RequestIterableType] = None,
        timeout: Optional[float] = None,
        metadata: Optional[Metadata] = None,
        credentials: Optional[grpc.CallCredentials] = None,
        wait_for_ready: Optional[bool] = None,
        compression: Optional[grpc.Compression] = None
    ) -> _base_call.StreamUnaryCall:

        metadata = self._init_metadata(metadata, compression)
        deadline = _timeout_to_deadline(timeout)

        if not self._interceptors:
            call = StreamUnaryCall(request_iterator, deadline, metadata,
                                   credentials, wait_for_ready, self._channel,
                                   self._method, self._request_serializer,
                                   self._response_deserializer, self._loop)
        else:
            call = InterceptedStreamUnaryCall(
                self._interceptors, request_iterator, deadline, metadata,
                credentials, wait_for_ready, self._channel, self._method,
                self._request_serializer, self._response_deserializer,
                self._loop)

        return call

class StreamStreamMultiCallable(_BaseMultiCallable,
                                _base_channel.StreamStreamMultiCallable):

    def __call__(
        self,
        request_iterator: Optional[RequestIterableType] = None,
        timeout: Optional[float] = None,
        metadata: Optional[Metadata] = None,
        credentials: Optional[grpc.CallCredentials] = None,
        wait_for_ready: Optional[bool] = None,
        compression: Optional[grpc.Compression] = None
    ) -> _base_call.StreamStreamCall:

        metadata = self._init_metadata(metadata, compression)
        deadline = _timeout_to_deadline(timeout)

        if not self._interceptors:
            call = StreamStreamCall(request_iterator, deadline, metadata,
                                    credentials, wait_for_ready, self._channel,
                                    self._method, self._request_serializer,
                                    self._response_deserializer, self._loop)
        else:
            call = InterceptedStreamStreamCall(
                self._interceptors, request_iterator, deadline, metadata,
                credentials, wait_for_ready, self._channel, self._method,
                self._request_serializer, self._response_deserializer,
                self._loop)

        return call

class Channel(_base_channel.Channel):
    _loop: asyncio.AbstractEventLoop
    _channel: cygrpc.AioChannel
    _unary_unary_interceptors: List[UnaryUnaryClientInterceptor]
    _unary_stream_interceptors: List[UnaryStreamClientInterceptor]
    _stream_unary_interceptors: List[StreamUnaryClientInterceptor]
    _stream_stream_interceptors: List[StreamStreamClientInterceptor]

    def __init__(self, target: str, options: ChannelArgumentType,
                 credentials: Optional[grpc.ChannelCredentials],
                 compression: Optional[grpc.Compression],
                 interceptors: Optional[Sequence[ClientInterceptor]]):
        """Constructor.

        Args:
          target: The target to which to connect.
          options: Configuration options for the channel.
          credentials: A cygrpc.ChannelCredentials or None.
          compression: An optional value indicating the compression method to be
            used over the lifetime of the channel.
          interceptors: An optional list of interceptors that would be used for
            intercepting any RPC executed with that channel.
        """
        self._unary_unary_interceptors = []
        self._unary_stream_interceptors = []
        self._stream_unary_interceptors = []
        self._stream_stream_interceptors = []

        if interceptors is not None:
            for interceptor in interceptors:
                if isinstance(interceptor, UnaryUnaryClientInterceptor):
                    self._unary_unary_interceptors.append(interceptor)
                elif isinstance(interceptor, UnaryStreamClientInterceptor):
                    self._unary_stream_interceptors.append(interceptor)
                elif isinstance(interceptor, StreamUnaryClientInterceptor):
                    self._stream_unary_interceptors.append(interceptor)
                elif isinstance(interceptor, StreamStreamClientInterceptor):
                    self._stream_stream_interceptors.append(interceptor)
                else:
                    raise ValueError(
                        "Interceptor {} must be ".format(interceptor) +
                        "{} or ".format(UnaryUnaryClientInterceptor.__name__) +
                        "{} or ".format(UnaryStreamClientInterceptor.__name__) +
                        "{} or ".format(StreamUnaryClientInterceptor.__name__) +
                        "{}. ".format(StreamStreamClientInterceptor.__name__))

        self._loop = cygrpc.get_working_loop()
        self._channel = cygrpc.AioChannel(
            _common.encode(target),
            _augment_channel_arguments(options, compression), credentials,
            self._loop)

    async def __aenter__(self):
        return self

    async def __aexit__(self, exc_type, exc_val, exc_tb):
        await self._close(None)

    async def _close(self, grace):  # pylint: disable=too-many-branches
        if self._channel.closed():
            return

        # No new calls will be accepted by the Cython channel.
        self._channel.closing()

        # Iterate through running tasks
        tasks = _all_tasks()
        calls = []
        call_tasks = []
        for task in tasks:
            try:
                stack = task.get_stack(limit=1)
            except AttributeError as attribute_error:
                # NOTE(lidiz) tl;dr: If the Task is created with a CPython
                # object, it will trigger AttributeError.
                #
                # In the global finalizer, the event loop schedules
                # a CPython PyAsyncGenAThrow object.
                # https://github.com/python/cpython/blob/00e45877e33d32bb61aa13a2033e3bba370bda4d/Lib/asyncio/base_events.py#L484
                #
                # However, the PyAsyncGenAThrow object is written in C and
                # failed to include the normal Python frame objects. Hence,
                # this exception is a false negative, and it is safe to ignore
                # the failure. It is fixed by https://github.com/python/cpython/pull/18669,
                # but not available until 3.9 or 3.8.3. So, we have to keep it
                # for a while.
                # TODO(lidiz) drop this hack after 3.8 deprecation
                if 'frame' in str(attribute_error):
                    continue
                else:
                    raise

            # If the Task is created by a C-extension, the stack will be empty.
            if not stack:
                continue

            # Locate ones created by `aio.Call`.
            frame = stack[0]
            candidate = frame.f_locals.get('self')
            if candidate:
                if isinstance(candidate, _base_call.Call):
                    if hasattr(candidate, '_channel'):
                        # For intercepted Call object
                        if candidate._channel is not self._channel:
                            continue
                    elif hasattr(candidate, '_cython_call'):
                        # For normal Call object
                        if candidate._cython_call._channel is not self._channel:
                            continue
                    else:
                        # Unidentified Call object
                        raise cygrpc.InternalError(
                            f'Unrecognized call object: {candidate}')

                    calls.append(candidate)
                    call_tasks.append(task)

        # If needed, try to wait for them to finish.
        # Call objects are not always awaitables.
        if grace and call_tasks:
            await asyncio.wait(call_tasks, timeout=grace)

        # Time to cancel existing calls.
        for call in calls:
            call.cancel()

        # Destroy the channel
        self._channel.close()

    async def close(self, grace: Optional[float] = None):
        await self._close(grace)

    def get_state(self,
                  try_to_connect: bool = False) -> grpc.ChannelConnectivity:
        result = self._channel.check_connectivity_state(try_to_connect)
        return _common.CYGRPC_CONNECTIVITY_STATE_TO_CHANNEL_CONNECTIVITY[result]

    async def wait_for_state_change(
        self,
        last_observed_state: grpc.ChannelConnectivity,
    ) -> None:
        assert await self._channel.watch_connectivity_state(
            last_observed_state.value[0], None)

    async def channel_ready(self) -> None:
        state = self.get_state(try_to_connect=True)
        while state != grpc.ChannelConnectivity.READY:
            await self.wait_for_state_change(state)
            state = self.get_state(try_to_connect=True)

    def unary_unary(
        self,
        method: str,
        request_serializer: Optional[SerializingFunction] = None,
        response_deserializer: Optional[DeserializingFunction] = None
    ) -> UnaryUnaryMultiCallable:
        return UnaryUnaryMultiCallable(self._channel, _common.encode(method),
                                       request_serializer,
                                       response_deserializer,
                                       self._unary_unary_interceptors,
                                       self._loop)

    def unary_stream(
        self,
        method: str,
        request_serializer: Optional[SerializingFunction] = None,
        response_deserializer: Optional[DeserializingFunction] = None
    ) -> UnaryStreamMultiCallable:
        return UnaryStreamMultiCallable(self._channel, _common.encode(method),
                                        request_serializer,
                                        response_deserializer,
                                        self._unary_stream_interceptors,
                                        self._loop)

    def stream_unary(
        self,
        method: str,
        request_serializer: Optional[SerializingFunction] = None,
        response_deserializer: Optional[DeserializingFunction] = None
    ) -> StreamUnaryMultiCallable:
        return StreamUnaryMultiCallable(self._channel, _common.encode(method),
                                        request_serializer,
                                        response_deserializer,
                                        self._stream_unary_interceptors,
                                        self._loop)

    def stream_stream(
        self,
        method: str,
        request_serializer: Optional[SerializingFunction] = None,
        response_deserializer: Optional[DeserializingFunction] = None
    ) -> StreamStreamMultiCallable:
        return StreamStreamMultiCallable(self._channel, _common.encode(method),
                                         request_serializer,
                                         response_deserializer,
                                         self._stream_stream_interceptors,
                                         self._loop)

def insecure_channel(
        target: str,
        options: Optional[ChannelArgumentType] = None,
        compression: Optional[grpc.Compression] = None,
        interceptors: Optional[Sequence[ClientInterceptor]] = None):
    """Creates an insecure asynchronous Channel to a server.

    Args:
      target: The server address
      options: An optional list of key-value pairs (:term:`channel_arguments`
        in gRPC Core runtime) to configure the channel.
      compression: An optional value indicating the compression method to be
        used over the lifetime of the channel. This is an EXPERIMENTAL option.
      interceptors: An optional sequence of interceptors that will be executed for
        any call executed with this channel.

    Returns:
      A Channel.
    """
    return Channel(target, () if options is None else options, None,
                   compression, interceptors)

def secure_channel(target: str,
                   credentials: grpc.ChannelCredentials,
                   options: Optional[ChannelArgumentType] = None,
                   compression: Optional[grpc.Compression] = None,
                   interceptors: Optional[Sequence[ClientInterceptor]] = None):
    """Creates a secure asynchronous Channel to a server.

    Args:
      target: The server address.
      credentials: A ChannelCredentials instance.
      options: An optional list of key-value pairs (:term:`channel_arguments`
        in gRPC Core runtime) to configure the channel.
      compression: An optional value indicating the compression method to be
        used over the lifetime of the channel. This is an EXPERIMENTAL option.
      interceptors: An optional sequence of interceptors that will be executed for
        any call executed with this channel.

    Returns:
      An aio.Channel.
    """
    return Channel(target, () if options is None else options,
                   credentials._credentials, compression, interceptors)
<commit_msg>[Aio] Ensure Core channel closes when deallocated (#29797)<commit_after># Copyright 2019 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Invocation-side implementation of gRPC Asyncio Python."""

import asyncio
import sys
from typing import Any, Iterable, List, Optional, Sequence

import grpc
from grpc import _common
from grpc import _compression
from grpc import _grpcio_metadata
from grpc._cython import cygrpc

from . import _base_call
from . import _base_channel
from ._call import StreamStreamCall
from ._call import StreamUnaryCall
from ._call import UnaryStreamCall
from ._call import UnaryUnaryCall
from ._interceptor import ClientInterceptor
from ._interceptor import InterceptedStreamStreamCall
from ._interceptor import InterceptedStreamUnaryCall
from ._interceptor import InterceptedUnaryStreamCall
from ._interceptor import InterceptedUnaryUnaryCall
from ._interceptor import StreamStreamClientInterceptor
from ._interceptor import StreamUnaryClientInterceptor
from ._interceptor import UnaryStreamClientInterceptor
from ._interceptor import UnaryUnaryClientInterceptor
from ._metadata import Metadata
from ._typing import ChannelArgumentType
from ._typing import DeserializingFunction
from ._typing import RequestIterableType
from ._typing import SerializingFunction
from ._utils import _timeout_to_deadline

_USER_AGENT = 'grpc-python-asyncio/{}'.format(_grpcio_metadata.__version__)

if sys.version_info[1] < 7:

    def _all_tasks() -> Iterable[asyncio.Task]:
        return asyncio.Task.all_tasks()
else:

    def _all_tasks() -> Iterable[asyncio.Task]:
        return asyncio.all_tasks()

def _augment_channel_arguments(base_options: ChannelArgumentType,
                               compression: Optional[grpc.Compression]):
    compression_channel_argument = _compression.create_channel_option(
        compression)
    user_agent_channel_argument = ((
        cygrpc.ChannelArgKey.primary_user_agent_string,
        _USER_AGENT,
    ),)
    return tuple(base_options
                ) + compression_channel_argument + user_agent_channel_argument

class _BaseMultiCallable:
    """Base class of all multi callable objects.

    Handles the initialization logic and stores common attributes.
    """
    _loop: asyncio.AbstractEventLoop
    _channel: cygrpc.AioChannel
    _method: bytes
    _request_serializer: SerializingFunction
    _response_deserializer: DeserializingFunction
    _interceptors: Optional[Sequence[ClientInterceptor]]
    _references: List[Any]
    _loop: asyncio.AbstractEventLoop

    # pylint: disable=too-many-arguments
    def __init__(
        self,
        channel: cygrpc.AioChannel,
        method: bytes,
        request_serializer: SerializingFunction,
        response_deserializer: DeserializingFunction,
        interceptors: Optional[Sequence[ClientInterceptor]],
        references: List[Any],
        loop: asyncio.AbstractEventLoop,
    ) -> None:
        self._loop = loop
        self._channel = channel
        self._method = method
        self._request_serializer = request_serializer
        self._response_deserializer = response_deserializer
        self._interceptors = interceptors
        self._references = references

    @staticmethod
    def _init_metadata(
            metadata: Optional[Metadata] = None,
            compression: Optional[grpc.Compression] = None) -> Metadata:
        """Based on the provided values for <metadata> or <compression> initialise the final
        metadata, as it should be used for the current call.
        """
        metadata = metadata or Metadata()
        if compression:
            metadata = Metadata(
                *_compression.augment_metadata(metadata, compression))
        return metadata

class UnaryUnaryMultiCallable(_BaseMultiCallable,
                              _base_channel.UnaryUnaryMultiCallable):

    def __call__(
        self,
        request: Any,
        *,
        timeout: Optional[float] = None,
        metadata: Optional[Metadata] = None,
        credentials: Optional[grpc.CallCredentials] = None,
        wait_for_ready: Optional[bool] = None,
        compression: Optional[grpc.Compression] = None
    ) -> _base_call.UnaryUnaryCall:

        metadata = self._init_metadata(metadata, compression)
        if not self._interceptors:
            call = UnaryUnaryCall(request, _timeout_to_deadline(timeout),
                                  metadata, credentials, wait_for_ready,
                                  self._channel, self._method,
                                  self._request_serializer,
                                  self._response_deserializer, self._loop)
        else:
            call = InterceptedUnaryUnaryCall(
                self._interceptors, request, timeout, metadata, credentials,
                wait_for_ready, self._channel, self._method,
                self._request_serializer, self._response_deserializer,
                self._loop)

        return call

class UnaryStreamMultiCallable(_BaseMultiCallable,
                               _base_channel.UnaryStreamMultiCallable):

    def __call__(
        self,
        request: Any,
        *,
        timeout: Optional[float] = None,
        metadata: Optional[Metadata] = None,
        credentials: Optional[grpc.CallCredentials] = None,
        wait_for_ready: Optional[bool] = None,
        compression: Optional[grpc.Compression] = None
    ) -> _base_call.UnaryStreamCall:

        metadata = self._init_metadata(metadata, compression)
        deadline = _timeout_to_deadline(timeout)

        if not self._interceptors:
            call = UnaryStreamCall(request, deadline, metadata, credentials,
                                   wait_for_ready, self._channel, self._method,
                                   self._request_serializer,
                                   self._response_deserializer, self._loop)
        else:
            call = InterceptedUnaryStreamCall(
                self._interceptors, request, deadline, metadata, credentials,
                wait_for_ready, self._channel, self._method,
                self._request_serializer, self._response_deserializer,
                self._loop)

        return call

class StreamUnaryMultiCallable(_BaseMultiCallable,
                               _base_channel.StreamUnaryMultiCallable):

    def __call__(
        self,
        request_iterator: Optional[RequestIterableType] = None,
        timeout: Optional[float] = None,
        metadata: Optional[Metadata] = None,
        credentials: Optional[grpc.CallCredentials] = None,
        wait_for_ready: Optional[bool] = None,
        compression: Optional[grpc.Compression] = None
    ) -> _base_call.StreamUnaryCall:

        metadata = self._init_metadata(metadata, compression)
        deadline = _timeout_to_deadline(timeout)

        if not self._interceptors:
            call = StreamUnaryCall(request_iterator, deadline, metadata,
                                   credentials, wait_for_ready, self._channel,
                                   self._method, self._request_serializer,
                                   self._response_deserializer, self._loop)
        else:
            call = InterceptedStreamUnaryCall(
                self._interceptors, request_iterator, deadline, metadata,
                credentials, wait_for_ready, self._channel, self._method,
                self._request_serializer, self._response_deserializer,
                self._loop)

        return call

class StreamStreamMultiCallable(_BaseMultiCallable,
                                _base_channel.StreamStreamMultiCallable):

    def __call__(
        self,
        request_iterator: Optional[RequestIterableType] = None,
        timeout: Optional[float] = None,
        metadata: Optional[Metadata] = None,
        credentials: Optional[grpc.CallCredentials] = None,
        wait_for_ready: Optional[bool] = None,
        compression: Optional[grpc.Compression] = None
    ) -> _base_call.StreamStreamCall:

        metadata = self._init_metadata(metadata, compression)
        deadline = _timeout_to_deadline(timeout)

        if not self._interceptors:
            call = StreamStreamCall(request_iterator, deadline, metadata,
                                    credentials, wait_for_ready, self._channel,
                                    self._method, self._request_serializer,
                                    self._response_deserializer, self._loop)
        else:
            call = InterceptedStreamStreamCall(
                self._interceptors, request_iterator, deadline, metadata,
                credentials, wait_for_ready, self._channel, self._method,
                self._request_serializer, self._response_deserializer,
                self._loop)

        return call

class Channel(_base_channel.Channel):
    _loop: asyncio.AbstractEventLoop
    _channel: cygrpc.AioChannel
    _unary_unary_interceptors: List[UnaryUnaryClientInterceptor]
    _unary_stream_interceptors: List[UnaryStreamClientInterceptor]
    _stream_unary_interceptors: List[StreamUnaryClientInterceptor]
    _stream_stream_interceptors: List[StreamStreamClientInterceptor]

    def __init__(self, target: str, options: ChannelArgumentType,
                 credentials: Optional[grpc.ChannelCredentials],
                 compression: Optional[grpc.Compression],
                 interceptors: Optional[Sequence[ClientInterceptor]]):
        """Constructor.

        Args:
          target: The target to which to connect.
          options: Configuration options for the channel.
          credentials: A cygrpc.ChannelCredentials or None.
          compression: An optional value indicating the compression method to be
            used over the lifetime of the channel.
          interceptors: An optional list of interceptors that would be used for
            intercepting any RPC executed with that channel.
        """
        self._unary_unary_interceptors = []
        self._unary_stream_interceptors = []
        self._stream_unary_interceptors = []
        self._stream_stream_interceptors = []

        if interceptors is not None:
            for interceptor in interceptors:
                if isinstance(interceptor, UnaryUnaryClientInterceptor):
                    self._unary_unary_interceptors.append(interceptor)
                elif isinstance(interceptor, UnaryStreamClientInterceptor):
                    self._unary_stream_interceptors.append(interceptor)
                elif isinstance(interceptor, StreamUnaryClientInterceptor):
                    self._stream_unary_interceptors.append(interceptor)
                elif isinstance(interceptor, StreamStreamClientInterceptor):
                    self._stream_stream_interceptors.append(interceptor)
                else:
                    raise ValueError(
                        "Interceptor {} must be ".format(interceptor) +
                        "{} or ".format(UnaryUnaryClientInterceptor.__name__) +
                        "{} or ".format(UnaryStreamClientInterceptor.__name__) +
                        "{} or ".format(StreamUnaryClientInterceptor.__name__) +
                        "{}. ".format(StreamStreamClientInterceptor.__name__))

        self._loop = cygrpc.get_working_loop()
        self._channel = cygrpc.AioChannel(
            _common.encode(target),
            _augment_channel_arguments(options, compression), credentials,
            self._loop)

    async def __aenter__(self):
        return self

    async def __aexit__(self, exc_type, exc_val, exc_tb):
        await self._close(None)

    async def _close(self, grace):  # pylint: disable=too-many-branches
        if self._channel.closed():
            return

        # No new calls will be accepted by the Cython channel.
        self._channel.closing()

        # Iterate through running tasks
        tasks = _all_tasks()
        calls = []
        call_tasks = []
        for task in tasks:
            try:
                stack = task.get_stack(limit=1)
            except AttributeError as attribute_error:
                # NOTE(lidiz) tl;dr: If the Task is created with a CPython
                # object, it will trigger AttributeError.
                #
                # In the global finalizer, the event loop schedules
                # a CPython PyAsyncGenAThrow object.
                # https://github.com/python/cpython/blob/00e45877e33d32bb61aa13a2033e3bba370bda4d/Lib/asyncio/base_events.py#L484
                #
                # However, the PyAsyncGenAThrow object is written in C and
                # failed to include the normal Python frame objects. Hence,
                # this exception is a false negative, and it is safe to ignore
                # the failure. It is fixed by https://github.com/python/cpython/pull/18669,
                # but not available until 3.9 or 3.8.3. So, we have to keep it
                # for a while.
                # TODO(lidiz) drop this hack after 3.8 deprecation
                if 'frame' in str(attribute_error):
                    continue
                else:
                    raise

            # If the Task is created by a C-extension, the stack will be empty.
            if not stack:
                continue

            # Locate ones created by `aio.Call`.
            frame = stack[0]
            candidate = frame.f_locals.get('self')
            if candidate:
                if isinstance(candidate, _base_call.Call):
                    if hasattr(candidate, '_channel'):
                        # For intercepted Call object
                        if candidate._channel is not self._channel:
                            continue
                    elif hasattr(candidate, '_cython_call'):
                        # For normal Call object
                        if candidate._cython_call._channel is not self._channel:
                            continue
                    else:
                        # Unidentified Call object
                        raise cygrpc.InternalError(
                            f'Unrecognized call object: {candidate}')

                    calls.append(candidate)
                    call_tasks.append(task)

        # If needed, try to wait for them to finish.
        # Call objects are not always awaitables.
        if grace and call_tasks:
            await asyncio.wait(call_tasks, timeout=grace)

        # Time to cancel existing calls.
        for call in calls:
            call.cancel()

        # Destroy the channel
        self._channel.close()

    async def close(self, grace: Optional[float] = None):
        await self._close(grace)

    def __del__(self):
        if hasattr(self, '_channel'):
            if not self._channel.closed():
                self._channel.close()

    def get_state(self,
                  try_to_connect: bool = False) -> grpc.ChannelConnectivity:
        result = self._channel.check_connectivity_state(try_to_connect)
        return _common.CYGRPC_CONNECTIVITY_STATE_TO_CHANNEL_CONNECTIVITY[result]

    async def wait_for_state_change(
        self,
        last_observed_state: grpc.ChannelConnectivity,
    ) -> None:
        assert await self._channel.watch_connectivity_state(
            last_observed_state.value[0], None)

    async def channel_ready(self) -> None:
        state = self.get_state(try_to_connect=True)
        while state != grpc.ChannelConnectivity.READY:
            await self.wait_for_state_change(state)
            state = self.get_state(try_to_connect=True)

    def unary_unary(
        self,
        method: str,
        request_serializer: Optional[SerializingFunction] = None,
        response_deserializer: Optional[DeserializingFunction] = None
    ) -> UnaryUnaryMultiCallable:
        return UnaryUnaryMultiCallable(self._channel, _common.encode(method),
                                       request_serializer,
                                       response_deserializer,
                                       self._unary_unary_interceptors, [self],
                                       self._loop)

    def unary_stream(
        self,
        method: str,
        request_serializer: Optional[SerializingFunction] = None,
        response_deserializer: Optional[DeserializingFunction] = None
    ) -> UnaryStreamMultiCallable:
        return UnaryStreamMultiCallable(self._channel, _common.encode(method),
                                        request_serializer,
                                        response_deserializer,
                                        self._unary_stream_interceptors, [self],
                                        self._loop)

    def stream_unary(
        self,
        method: str,
        request_serializer: Optional[SerializingFunction] = None,
        response_deserializer: Optional[DeserializingFunction] = None
    ) -> StreamUnaryMultiCallable:
        return StreamUnaryMultiCallable(self._channel, _common.encode(method),
                                        request_serializer,
                                        response_deserializer,
                                        self._stream_unary_interceptors, [self],
                                        self._loop)

    def stream_stream(
        self,
        method: str,
        request_serializer: Optional[SerializingFunction] = None,
        response_deserializer: Optional[DeserializingFunction] = None
    ) -> StreamStreamMultiCallable:
        return StreamStreamMultiCallable(self._channel, _common.encode(method),
                                         request_serializer,
                                         response_deserializer,
                                         self._stream_stream_interceptors,
                                         [self], self._loop)

def insecure_channel(
        target: str,
        options: Optional[ChannelArgumentType] = None,
        compression: Optional[grpc.Compression] = None,
        interceptors: Optional[Sequence[ClientInterceptor]] = None):
    """Creates an insecure asynchronous Channel to a server.

    Args:
      target: The server address
      options: An optional list of key-value pairs (:term:`channel_arguments`
        in gRPC Core runtime) to configure the channel.
      compression: An optional value indicating the compression method to be
        used over the lifetime of the channel. This is an EXPERIMENTAL option.
      interceptors: An optional sequence of interceptors that will be executed for
        any call executed with this channel.

    Returns:
      A Channel.
    """
    return Channel(target, () if options is None else options, None,
                   compression, interceptors)

def secure_channel(target: str,
                   credentials: grpc.ChannelCredentials,
                   options: Optional[ChannelArgumentType] = None,
                   compression: Optional[grpc.Compression] = None,
                   interceptors: Optional[Sequence[ClientInterceptor]] = None):
    """Creates a secure asynchronous Channel to a server.

    Args:
      target: The server address.
      credentials: A ChannelCredentials instance.
      options: An optional list of key-value pairs (:term:`channel_arguments`
        in gRPC Core runtime) to configure the channel.
      compression: An optional value indicating the compression method to be
        used over the lifetime of the channel. This is an EXPERIMENTAL option.
      interceptors: An optional sequence of interceptors that will be executed for
        any call executed with this channel.

    Returns:
      An aio.Channel.
    """
    return Channel(target, () if options is None else options,
                   credentials._credentials, compression, interceptors)
<|endoftext|>


<commit_before>import re
import io
import zipfile
import base64
import os
import os.path
import errno
import collections
import itertools

import requests
import yaml

BASE_URL = 'https://api.github.com/repos/{user}/{repo}/'
extension_index = {}
filename_index = {}
lang_data = None

def get_repo(user, repo):
    url = BASE_URL.format(user=user, repo=repo)
    get_data(url)
    write_data()

def get_data(url):
    global lang_data
    lang_data = collections.defaultdict(list)
    data = requests.get(url + 'zipball/master')
    data.raise_for_status()
    archive = zipfile.ZipFile(io.BytesIO(data.content))
    for name in archive.namelist():
        base = os.path.basename(name)
        if base:
            lang = filename_index.get(base)
            if not lang:
                ext = os.path.splitext(base)[1]
                if ext:
                    lang = extension_index.get(ext)
            if lang:
                with archive.open(name) as f:
                    lang_data[lang].extend(f.readlines())

def write_data():
    for lang, lines in lang_data.items():
        with open(os.path.join('corpus', lang), 'a') as f:
            f.writelines([l.decode(errors='ignore') for l in lines])

def get_important_repos():
    important_repos = requests.get('https://github.com/repositories')
    important_repos.raise_for_status()
    url_matches = re.finditer(r'\2\n',
                            important_repos.text)
    results = ((match.group(1), match.group(2)) for match in url_matches)
    return itertools.islice(results, 0, 100, 2)

def init_linguist():
    '''get GitHub's programming language identification database'''
    r = requests.get(BASE_URL.format(user='github', repo='linguist') +
                     'contents/lib/linguist/languages.yml')
    r.raise_for_status()
    content = base64.b64decode(r.json['content'].encode('utf-8'))
    linguist = yaml.load(content)
    build_index(linguist)

def build_index(linguist):
    '''create two reverse indexes for extensions and filenames to language'''
    global extension_index
    global filename_index
    for lang, data in linguist.items():
        exts = {data.get('primary_extension')}
        exts = exts.union(data.get('extensions', {}))
        for ext in exts.union(data.get('overrides', {})):
            extension_index[ext] = lang
        for name in data.get('filenames', []):
            filename_index[name] = lang

def main():
    try:
        os.mkdir('corpus')
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise e
        else:
            print('the "corpus" directory already exists,',
                  'all data will be appended to the existing files')
    init_linguist()
    for (user, repo) in get_important_repos():
        print('processing repo: {user}/{repo}'.format(user=user, repo=repo))
        get_repo(user, repo)

if __name__ == '__main__':
    main()
<commit_msg>cleanup<commit_after>import re
import io
import zipfile
import base64
import os
import os.path
import collections
import itertools
import tempfile

import requests
import yaml

BASE_URL = 'https://api.github.com/repos/{user}/{repo}/'
extension_index = {}
filename_index = {}

def process_repo(user, repo, corpus_dir):
    '''append files of user/repo to language files in corpus_dir'''
    print('processing repo: {user}/{repo}'.format(user=user, repo=repo))
    url = BASE_URL.format(user=user, repo=repo)
    data = get_data(url)
    write_data(data, corpus_dir)

def get_data(url):
    '''return dictionary lang -> lines for the github project given as url'''
    lang_data = collections.defaultdict(list)

    data = requests.get(url + 'zipball/master')
    try:
        data.raise_for_status()
    except requests.exceptions.HTTPError as e:
        print('got ' + str(e) + ' when accessing ' + url)
    else:
        archive = zipfile.ZipFile(io.BytesIO(data.content))
        update_lang_data(lang_data, archive)

    return lang_data

def update_lang_data(lang_data, archive):
    '''for each recognized file in archive, add its lines to lang_data'''
    for name in archive.namelist():
        base = os.path.basename(name)
        if base:
            lang = identify_lang(base)
            if lang:
                with archive.open(name) as f:
                    lang_data[lang].extend(f.readlines())

def identify_lang(filename):
    lang = filename_index.get(filename)
    if not lang:
        ext = os.path.splitext(filename)[1]
        if ext:
            lang = extension_index.get(ext)
    return lang

def write_data(lang_data, corpus_dir):
    '''append lines stored in lang_data to files in corpus_dir
    those files are named by language'''
    for lang, lines in lang_data.items():
        with open(os.path.join(corpus_dir, lang), 'a') as f:
            f.writelines([l.decode(errors='ignore') for l in lines])

def get_important_repos(num=25):
    '''return the repositories listed on github "important" page
    as a generator of pairs (user, repo)'''
    important_repos = requests.get('https://github.com/repositories')
    important_repos.raise_for_status()
    url_matches = re.finditer(r'\2\n',
                            important_repos.text)
    results = ((match.group(1), match.group(2)) for match in url_matches)
    return itertools.islice(results, 0, 2 * num, 2)

def init_linguist():
    '''get GitHub's programming language identification database'''
    r = requests.get(BASE_URL.format(user='github', repo='linguist') +
                     'contents/lib/linguist/languages.yml')
    r.raise_for_status()
    content = base64.b64decode(r.json['content'].encode('utf-8'))
    linguist = yaml.load(content)
    build_index(linguist)

def build_index(linguist):
    '''create two reverse indexes for extensions and filenames to language'''
    global extension_index
    global filename_index
    for lang, data in linguist.items():
        exts = {data.get('primary_extension')}
        exts = exts.union(data.get('extensions', {}))
        for ext in exts.union(data.get('overrides', {})):
            extension_index[ext] = lang
        for name in data.get('filenames', []):
            filename_index[name] = lang

def main():
    init_linguist()
    with tempfile.TemporaryDirectory() as corpus_dir:
        for (user, repo) in get_important_repos():
            process_repo(user, repo, corpus_dir)
        with zipfile.ZipFile('corpus.zip', 'w',
                             compression=zipfile.ZIP_DEFLATED) as myzip:
            for langfile in os.listdir(corpus_dir):
                myzip.write(os.path.join(corpus_dir, langfile),
                            os.path.join('corpus', langfile))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""
Utilities to generate and plot sensor data sequence classification
experiments. """

import csv
import os
import math
import random

import matplotlib.pyplot as plt

def plotSensorData(expResults):
  """
  Plot several sensor data CSV recordings and highlights the sequences.
  @param expResults: (list of dict) list of results for each experiment
  """

  categoryColors = ['grey', 'b', 'r', 'y']
  plt.figure()

  for expResult in expResults:

    timesteps = []
    data = []
    labels = []
    categoriesLabelled = []

    filePath = expResult['inputFilePath']
    with open(filePath, 'rb') as f:
      reader = csv.reader(f)
      headers = reader.next()

      # skip the 2 first rows
      reader.next()
      reader.next()

      for i, values in enumerate(reader):
        record = dict(zip(headers, values))
        timesteps.append(i)
        data.append(record['y'])
        labels.append(int(record['label']))

      plt.subplot(len(expResults), 1, expResults.index(expResult) + 1)
      plt.plot(timesteps, data, 'xb-', label='signal')

      previousLabel = labels[0]
      start = 0
      labelCount = 0
      numPoints = len(labels)
      for label in labels:

        if previousLabel != label or labelCount == numPoints - 1:

          categoryColor = categoryColors[previousLabel]
          if categoryColor not in categoriesLabelled:
            if previousLabel > 0:
              labelLegend = 'sequence %s' % previousLabel
            else:
              labelLegend = 'noise'
            categoriesLabelled.append(categoryColor)
          else:
            labelLegend = None

          end = labelCount
          plt.axvspan(start, end, facecolor=categoryColor, alpha=0.5,
                      label=labelLegend)
          start = end
          previousLabel = label

        labelCount += 1

      plt.xlim(xmin=0, xmax=len(timesteps))

      title = cleanTitle(expResult)
      plt.title(title)

      plt.legend()

  plt.show()

def cleanTitle(expResult):
  """
  Clean up chart title based on expResult info
  :param expResult: (dict) results of the experiment.
  :return title: (str) cleaned up title
  """
  # title = expResult['expId'].split('_')
  # cleanTitle = [title[0] + ' signal']
  # for word in title[1:]:
  #   if not 'False' in word:
  #     if 'True' in word:
  #       cleanTitle.append(word[:2])
  #     else:
  #       cleanTitle.append(word)
  #
  # title = ' + '.join(cleanTitle)
  title = ''
  return title

def generateSensorData(signalType,
                       dataDir,
                       numPhases,
                       numReps,
                       signalMean,
                       signalAmplitude,
                       numCategories,
                       noiseAmplitude,
                       noiseLength):
  """
  Generate the artificial sensor data.
  @param signalType: (str) can be one of the following:
  - 'sine': Sine wave with a different frequency per category.
  - 'binary': periodic binary signal with a different period per category.
  - 'triangle': triangular signal with a different sampling rate per category.
  @param dataDir: (str) directory where to save the CSV files
  @param numPhases: (int) number of phases to train and test. E.g: 4 phases
    to train and 1 to test: (1) SP, (2) TM, (3) TP, (4) Classifier, (5) Test.
  @param numReps: (int) Number of time each phase repeats.
  @param signalMean: (float) mean of the signal to generate
  @param signalAmplitude: (float) amplitude of the signal to generate
  @param numCategories: (int) number of categories labels
  @param noiseAmplitude: (float) amplitude of the white noise
  @param noiseLength: (int) number of noisy points at the beginning of each seq.
  @return expSetup: (dict) setup for each experiment
  """

  # some info about the experiment setup
  expSetup = {
    'signalType': signalType,
    'signalMean': signalMean,
    'signalAmplitude': signalAmplitude,
    'numCategories': numCategories,
    'noiseAmplitude': noiseAmplitude,
    'noiseLength': noiseLength,
    'numPhases': numPhases,
    'numReps': numReps,
  }

  if signalType == 'sine':
    signal_generator = sine_wave_generator
  elif signalType == 'binary':
    signal_generator = binary_signal_generator
  elif signalType == 'triangle':
    signal_generator = triangular_signal_generator
  else:
    raise ValueError('Signal type can only be "sine", "triangle" or "step"')

  # make sure the directory exist. if not, create it.
  if not os.path.exists(dataDir):
    os.makedirs(dataDir)

  filePath = "%s/%s_ampl=%s_mean=%s_noise=%s.csv" % (dataDir, signalType,
                                                     signalAmplitude,
                                                     signalMean,
                                                     noiseAmplitude)
  with open(filePath, "wb") as f:
    writer = csv.writer(f)
    writer.writerow(["x", "y", "label"])
    writer.writerow(["float", "float", "int"])
    writer.writerow(["", "", "C"])  # C is for category.
    # WARNING: if the C flag is forgotten in the dataset, then all records will
    #  be arbitrarily put in the same category (i.e. category 0). So make sure
    # to have the C flag -- otherwise you'll get 100% classification accuracy
    # regardless of the input data :-P

    sequenceLength, numPoints = signal_generator(writer,
                                                 numPhases,
                                                 numReps,
                                                 signalMean,
                                                 noiseAmplitude,
                                                 signalAmplitude,
                                                 numCategories,
                                                 noiseLength)

  expSetup['sequenceLength'] = sequenceLength
  expSetup['numPoints'] = numPoints
  expSetup['inputFilePath'] = filePath

  return expSetup

def sine_wave_generator(writer,
                        numPhases,
                        numReps,
                        signalMean,
                        noiseAmplitude,
                        signalAmplitude,
                        numCategories,
                        noiseLength=10):
  """
  Generate the artificial sensor data.
  @param writer: (csv.CSVWriter) csv file writer.
  @param numPhases: (int) number of phases to train and test. E.g: 4 phases
    to train and 1 to test: (1) SP, (2) TM, (3) TP, (4) Classifier, (5) Test.
  @param numReps: (int) Number of time each phase repeats.
  @param signalMean: (float) mean of the signal to generate
  @param noiseAmplitude: (float) amplitude of the white noise
  @param signalAmplitude: (float) amplitude of the signal to generate
  @param numCategories: (int) number of categories labels
  @param noiseLength: (int) number of noisy points at the beginning of each seq.
  """

  signalPeriod = 20
  sequenceLength = signalPeriod * 4
  numPoints = numReps * numPhases * sequenceLength * numCategories

  endOfSequence = sequenceLength
  label = 1
  offset = 0
  for i in range(numPoints):

    noise = noiseAmplitude * random.random()

    if i == endOfSequence:
      endOfSequence += sequenceLength
      if label == numCategories:
        label = 1
      else:
        label += 1

      for j in range(noiseLength):
        offset += 1
        t = i + offset
        m1 = random.random() * signalAmplitude
        writer.writerow([t, m1, 0])

    signal_modifier = (label ** 2 + 1)

    t = i + offset
    sig = math.sin(signal_modifier * (t * math.pi) / signalPeriod)
    m1 = signal_modifier * signalMean + signalAmplitude * sig + noise

    writer.writerow([t, m1, label])

  numPoints = offset + numPoints
  assert offset == (numReps * numPhases * numCategories - 1) * noiseLength
  return sequenceLength, numPoints

def binary_signal_generator(writer,
                            numPhases,
                            numReps,
                            signalMean,
                            noiseAmplitude,
                            signalAmplitude,
                            numCategories,
                            noiseLength=10):
  """
  Generate the artificial sensor data.
  @param writer: (csv.CSVWriter) csv file writer.
  @param numPhases: (int) number of phases to train and test. E.g: 4 phases
    to train and 1 to test: (1) SP, (2) TM, (3) TP, (4) Classifier, (5) Test.
  @param numReps: (int) Number of time each phase repeats.
  @param signalMean: (float) mean of the signal to generate
  @param noiseAmplitude: (float) amplitude of the white noise
  @param signalAmplitude: (float) amplitude of the signal to generate
  @param numCategories: (int) number of categories labels
  @param noiseLength: (int) number of noisy points at the beginning of each seq.
  """

  # if numCategories = 3, then sequenceLength = 4 * 3 * 2 = 24
  # if numCategories = 2, then sequenceLength = 3 * 2 = 6
  minSequenceLength = 1
  for cat in range(numCategories + 1)[1:]:
    minSequenceLength *= cat
  sequenceLength = minSequenceLength * 10
  numPoints = numReps * numPhases * sequenceLength * numCategories

  endOfSequence = sequenceLength
  label = 1
  periodCounter = [0 for _ in range(numCategories)]
  sig = 0
  offset = 0
  for i in range(numPoints):
    noise = noiseAmplitude * random.random()

    if i == endOfSequence:
      endOfSequence += sequenceLength
      if label == numCategories:
        label = 1
      else:
        label += 1

      for j in range(noiseLength):
        offset += 1
        t = i + offset
        m1 = random.random() * signalAmplitude
        writer.writerow([t, m1, 0])

    amplitude_modifier = float(label) ** 2
    m1 = amplitude_modifier * signalMean + signalAmplitude * sig + noise
    writer.writerow([i, m1, label])

    if periodCounter[label - 1] == label-1:
      periodCounter = [0 for _ in range(numCategories)]
      if sig == 0:
        sig = 1
      else:
        sig = 0
    else:
      periodCounter[label - 1] += 1

  assert offset == (numReps * numPhases * numCategories - 1) * noiseLength
  return sequenceLength, numPoints

def triangular_signal_generator(writer,
                                numPhases,
                                numReps,
                                signalMean,
                                noiseAmplitude,
                                signalAmplitude,
                                numCategories,
                                noiseLength=10):
  """
  Generate the artificial sensor data.
  @param writer: (csv.CSVWriter) csv file writer.
  @param numPhases: (int) number of phases to train and test. E.g: 4 phases
    to train and 1 to test: (1) SP, (2) TM, (3) TP, (4) Classifier, (5) Test.
  @param numReps: (int) Number of time each phase repeats.
  @param signalMean: (float) mean of the signal to generate
  @param noiseAmplitude: (float) amplitude of the white noise
  @param signalAmplitude: (float) amplitude of the signal to generate
  @param numCategories: (int) number of categories labels
  @param noiseLength: (int) number of noisy points at the beginning of each seq.
  """

  # if numCategories = 3, then sequenceLength = 4 * 3 * 2 = 24
  # if numCategories = 2, then sequenceLength = 3 * 2 = 6
  minSequenceLength = math.factorial(numCategories + 1)
  sequenceLength = minSequenceLength * 3
  numPoints = numReps * numPhases * sequenceLength * numCategories

  endOfSequence = sequenceLength
  label = 1
  offset = 0
  for i in range(numPoints):

    noise = noiseAmplitude * random.random()

    if i == endOfSequence:
      endOfSequence += sequenceLength
      if label == numCategories:
        label = 1
      else:
        label += 1

      for j in range(noiseLength):
        offset += 1
        t = i + offset
        m1 = random.random() * signalAmplitude
        writer.writerow([t, m1, 0])

    x = i
    mod = int(label) + 2
    # if label = 0, then mod = 2, so sig = 0 or 1
    # if label = 1, then mod = 3, so sig = 0 or 0.5 or 1
    # if label = 2, then mod = 4, so sig = 0 or 0.33 or 0.66 or 1
    sig = i % mod / (mod - 1.0)

    amplitude_modifier = float(label) ** 2
    m1 = amplitude_modifier * signalMean + signalAmplitude * sig + noise

    writer.writerow([x, m1, label])

  assert offset == (numReps * numPhases * numCategories - 1) * noiseLength
  return sequenceLength, numPoints
<commit_msg>cleaned up function<commit_after>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""
Utilities to generate and plot sensor data sequence classification
experiments. """

import csv
import os
import math
import random

import matplotlib.pyplot as plt

def plotSensorData(expResults):
  """
  Plot several sensor data CSV recordings and highlights the sequences.
  @param expResults: (list of dict) list of results for each experiment
  """

  categoryColors = ['grey', 'b', 'r', 'y']
  plt.figure()

  for expResult in expResults:

    timesteps = []
    data = []
    labels = []
    categoriesLabelled = []

    filePath = expResult['inputFilePath']
    with open(filePath, 'rb') as f:
      reader = csv.reader(f)
      headers = reader.next()

      # skip the 2 first rows
      reader.next()
      reader.next()

      for i, values in enumerate(reader):
        record = dict(zip(headers, values))
        timesteps.append(i)
        data.append(record['y'])
        labels.append(int(record['label']))

      plt.subplot(len(expResults), 1, expResults.index(expResult) + 1)
      plt.plot(timesteps, data, 'xb-', label='signal')

      previousLabel = labels[0]
      start = 0
      labelCount = 0
      numPoints = len(labels)
      for label in labels:

        if previousLabel != label or labelCount == numPoints - 1:

          categoryColor = categoryColors[previousLabel]
          if categoryColor not in categoriesLabelled:
            if previousLabel > 0:
              labelLegend = 'sequence %s' % previousLabel
            else:
              labelLegend = 'noise'
            categoriesLabelled.append(categoryColor)
          else:
            labelLegend = None

          end = labelCount
          plt.axvspan(start, end, facecolor=categoryColor, alpha=0.5,
                      label=labelLegend)
          start = end
          previousLabel = label

        labelCount += 1

      plt.xlim(xmin=0, xmax=len(timesteps))

      title = cleanTitle(expResult)
      plt.title(title)

      plt.legend()

  plt.show()

def cleanTitle(expResult):
  """
  Clean up chart title based on expResult info
  :param expResult: (dict) results of the experiment.
  :return title: (str) cleaned up title
  """
  expId = expResult['inputFilePath'].split('/')[-1][:-4]
  title = expId.split('_')
  cleanTitle = [title[0] + ' signal']
  for word in title[1:]:
    if not 'False' in word:
      if 'True' in word:
        cleanTitle.append(word[:2])
      else:
        cleanTitle.append(word)

  title = ' + '.join(cleanTitle)

  return title

def generateSensorData(signalType,
                       dataDir,
                       numPhases,
                       numReps,
                       signalMean,
                       signalAmplitude,
                       numCategories,
                       noiseAmplitude,
                       noiseLength):
  """
  Generate the artificial sensor data.
  @param signalType: (str) can be one of the following:
  - 'sine': Sine wave with a different frequency per category.
  - 'binary': periodic binary signal with a different period per category.
  - 'triangle': triangular signal with a different sampling rate per category.
  @param dataDir: (str) directory where to save the CSV files
  @param numPhases: (int) number of phases to train and test. E.g: 4 phases
    to train and 1 to test: (1) SP, (2) TM, (3) TP, (4) Classifier, (5) Test.
  @param numReps: (int) Number of time each phase repeats.
  @param signalMean: (float) mean of the signal to generate
  @param signalAmplitude: (float) amplitude of the signal to generate
  @param numCategories: (int) number of categories labels
  @param noiseAmplitude: (float) amplitude of the white noise
  @param noiseLength: (int) number of noisy points at the beginning of each seq.
  @return expSetup: (dict) setup for each experiment
  """

  # some info about the experiment setup
  expSetup = {
    'signalType': signalType,
    'signalMean': signalMean,
    'signalAmplitude': signalAmplitude,
    'numCategories': numCategories,
    'noiseAmplitude': noiseAmplitude,
    'noiseLength': noiseLength,
    'numPhases': numPhases,
    'numReps': numReps,
  }

  if signalType == 'sine':
    signal_generator = sine_wave_generator
  elif signalType == 'binary':
    signal_generator = binary_signal_generator
  elif signalType == 'triangle':
    signal_generator = triangular_signal_generator
  else:
    raise ValueError('Signal type can only be "sine", "triangle" or "step"')

  # make sure the directory exist. if not, create it.
  if not os.path.exists(dataDir):
    os.makedirs(dataDir)

  filePath = "%s/%s_ampl=%s_mean=%s_noise=%s.csv" % (dataDir, signalType,
                                                     signalAmplitude,
                                                     signalMean,
                                                     noiseAmplitude)
  with open(filePath, "wb") as f:
    writer = csv.writer(f)
    writer.writerow(["x", "y", "label"])
    writer.writerow(["float", "float", "int"])
    writer.writerow(["", "", "C"])  # C is for category.
    # WARNING: if the C flag is forgotten in the dataset, then all records will
    #  be arbitrarily put in the same category (i.e. category 0). So make sure
    # to have the C flag -- otherwise you'll get 100% classification accuracy
    # regardless of the input data :-P

    sequenceLength, numPoints = signal_generator(writer,
                                                 numPhases,
                                                 numReps,
                                                 signalMean,
                                                 noiseAmplitude,
                                                 signalAmplitude,
                                                 numCategories,
                                                 noiseLength)

  expSetup['sequenceLength'] = sequenceLength
  expSetup['numPoints'] = numPoints
  expSetup['inputFilePath'] = filePath

  return expSetup

def sine_wave_generator(writer,
                        numPhases,
                        numReps,
                        signalMean,
                        noiseAmplitude,
                        signalAmplitude,
                        numCategories,
                        noiseLength=10):
  """
  Generate the artificial sensor data.
  @param writer: (csv.CSVWriter) csv file writer.
  @param numPhases: (int) number of phases to train and test. E.g: 4 phases
    to train and 1 to test: (1) SP, (2) TM, (3) TP, (4) Classifier, (5) Test.
  @param numReps: (int) Number of time each phase repeats.
  @param signalMean: (float) mean of the signal to generate
  @param noiseAmplitude: (float) amplitude of the white noise
  @param signalAmplitude: (float) amplitude of the signal to generate
  @param numCategories: (int) number of categories labels
  @param noiseLength: (int) number of noisy points at the beginning of each seq.
  """

  signalPeriod = 20
  sequenceLength = signalPeriod * 4
  numPoints = numReps * numPhases * sequenceLength * numCategories

  endOfSequence = sequenceLength
  label = 1
  offset = 0
  for i in range(numPoints):

    noise = noiseAmplitude * random.random()

    if i == endOfSequence:
      endOfSequence += sequenceLength
      if label == numCategories:
        label = 1
      else:
        label += 1

      for j in range(noiseLength):
        offset += 1
        t = i + offset
        m1 = random.random() * signalAmplitude
        writer.writerow([t, m1, 0])

    signal_modifier = (label ** 2 + 1)

    t = i + offset
    sig = math.sin(signal_modifier * (t * math.pi) / signalPeriod)
    m1 = signal_modifier * signalMean + signalAmplitude * sig + noise

    writer.writerow([t, m1, label])

  numPoints = offset + numPoints
  assert offset == (numReps * numPhases * numCategories - 1) * noiseLength
  return sequenceLength, numPoints

def binary_signal_generator(writer,
                            numPhases,
                            numReps,
                            signalMean,
                            noiseAmplitude,
                            signalAmplitude,
                            numCategories,
                            noiseLength=10):
  """
  Generate the artificial sensor data.
  @param writer: (csv.CSVWriter) csv file writer.
  @param numPhases: (int) number of phases to train and test. E.g: 4 phases
    to train and 1 to test: (1) SP, (2) TM, (3) TP, (4) Classifier, (5) Test.
  @param numReps: (int) Number of time each phase repeats.
  @param signalMean: (float) mean of the signal to generate
  @param noiseAmplitude: (float) amplitude of the white noise
  @param signalAmplitude: (float) amplitude of the signal to generate
  @param numCategories: (int) number of categories labels
  @param noiseLength: (int) number of noisy points at the beginning of each seq.
  """

  # if numCategories = 3, then sequenceLength = 4 * 3 * 2 = 24
  # if numCategories = 2, then sequenceLength = 3 * 2 = 6
  minSequenceLength = 1
  for cat in range(numCategories + 1)[1:]:
    minSequenceLength *= cat
  sequenceLength = minSequenceLength * 10
  numPoints = numReps * numPhases * sequenceLength * numCategories

  endOfSequence = sequenceLength
  label = 1
  periodCounter = [0 for _ in range(numCategories)]
  sig = 0
  offset = 0
  for i in range(numPoints):
    noise = noiseAmplitude * random.random()

    if i == endOfSequence:
      endOfSequence += sequenceLength
      if label == numCategories:
        label = 1
      else:
        label += 1

      for j in range(noiseLength):
        offset += 1
        t = i + offset
        m1 = random.random() * signalAmplitude
        writer.writerow([t, m1, 0])

    amplitude_modifier = float(label) ** 2
    m1 = amplitude_modifier * signalMean + signalAmplitude * sig + noise
    writer.writerow([i, m1, label])

    if periodCounter[label - 1] == label-1:
      periodCounter = [0 for _ in range(numCategories)]
      if sig == 0:
        sig = 1
      else:
        sig = 0
    else:
      periodCounter[label - 1] += 1

  assert offset == (numReps * numPhases * numCategories - 1) * noiseLength
  return sequenceLength, numPoints

def triangular_signal_generator(writer,
                                numPhases,
                                numReps,
                                signalMean,
                                noiseAmplitude,
                                signalAmplitude,
                                numCategories,
                                noiseLength=10):
  """
  Generate the artificial sensor data.
  @param writer: (csv.CSVWriter) csv file writer.
  @param numPhases: (int) number of phases to train and test. E.g: 4 phases
    to train and 1 to test: (1) SP, (2) TM, (3) TP, (4) Classifier, (5) Test.
  @param numReps: (int) Number of time each phase repeats.
  @param signalMean: (float) mean of the signal to generate
  @param noiseAmplitude: (float) amplitude of the white noise
  @param signalAmplitude: (float) amplitude of the signal to generate
  @param numCategories: (int) number of categories labels
  @param noiseLength: (int) number of noisy points at the beginning of each seq.
  """

  # if numCategories = 3, then sequenceLength = 4 * 3 * 2 = 24
  # if numCategories = 2, then sequenceLength = 3 * 2 = 6
  minSequenceLength = math.factorial(numCategories + 1)
  sequenceLength = minSequenceLength * 3
  numPoints = numReps * numPhases * sequenceLength * numCategories

  endOfSequence = sequenceLength
  label = 1
  offset = 0
  for i in range(numPoints):

    noise = noiseAmplitude * random.random()

    if i == endOfSequence:
      endOfSequence += sequenceLength
      if label == numCategories:
        label = 1
      else:
        label += 1

      for j in range(noiseLength):
        offset += 1
        t = i + offset
        m1 = random.random() * signalAmplitude
        writer.writerow([t, m1, 0])

    x = i
    mod = int(label) + 2
    # if label = 0, then mod = 2, so sig = 0 or 1
    # if label = 1, then mod = 3, so sig = 0 or 0.5 or 1
    # if label = 2, then mod = 4, so sig = 0 or 0.33 or 0.66 or 1
    sig = i % mod / (mod - 1.0)

    amplitude_modifier = float(label) ** 2
    m1 = amplitude_modifier * signalMean + signalAmplitude * sig + noise

    writer.writerow([x, m1, label])

  assert offset == (numReps * numPhases * numCategories - 1) * noiseLength
  return sequenceLength, numPoints
<|endoftext|>


<commit_before>from smart.models import *
from base import utils
from django.http import HttpResponse
from smart.models.ontology_url_patterns import CallMapper

@CallMapper.register(client_method_name="put_user_preferences")
def preferences_put (request, user_id, pha_email):
    try:
        ct = utils.get_content_type(request).lower().split(';')[0]
        if (not ct) or len(ct) == 0 or ct == "none": ct = "text/plain"
    except:
        ct = "text/plain"

    p = fetch_preferences (user_id, pha_email)
    p.data = request.raw_post_data
    p.mime = ct
    p.save()
    return HttpResponse("ok")

@CallMapper.register(client_method_name="get_user_preferences")
def preferences_get (request, user_id, pha_email):
    p = fetch_preferences (user_id, pha_email)
    return HttpResponse(p.data, mimetype=p.mime)

@CallMapper.register(client_method_name="delete_user_preferences")
def preferences_delete(request, user_id, pha_email):
    p = fetch_preferences (user_id, pha_email)
    p.delete()
    return HttpResponse("ok")

def resolve_account_pha (user_id, pha_email):
    account = None
    pha = None

    if user_id != None:
        try:
            account = Account.objects.get(email=user_id)
        except Account.DoesNotExist:
            pass

    if pha_email != None:
        try:
            pha = PHA.objects.get(email=pha_email)
        except PHA.DoesNotExist:
            pass

    return account, pha

def fetch_preferences(user_id, pha_email):
    account, pha = resolve_account_pha (user_id, pha_email)
    return Preferences.objects.get_or_create(account=account, pha=pha, defaults={"data": "", "mime": "text/plain"})[0]
<commit_msg>Updated return type for put_user_preferences.  Closes chb/smart_project_management#64<commit_after>from smart.models import *
from base import utils
from django.http import HttpResponse
from smart.models.ontology_url_patterns import CallMapper

@CallMapper.register(client_method_name="put_user_preferences")
def preferences_put (request, user_id, pha_email):
    try:
        ct = utils.get_content_type(request).lower().split(';')[0]
        if (not ct) or len(ct) == 0 or ct == "none": ct = "text/plain"
    except:
        ct = "text/plain"

    p = fetch_preferences (user_id, pha_email)
    p.data = request.raw_post_data
    p.mime = ct
    p.save()
    return HttpResponse(p.data, mimetype=p.mime)

@CallMapper.register(client_method_name="get_user_preferences")
def preferences_get (request, user_id, pha_email):
    p = fetch_preferences (user_id, pha_email)
    return HttpResponse(p.data, mimetype=p.mime)

@CallMapper.register(client_method_name="delete_user_preferences")
def preferences_delete(request, user_id, pha_email):
    p = fetch_preferences (user_id, pha_email)
    p.delete()
    return HttpResponse("ok")

def resolve_account_pha (user_id, pha_email):
    account = None
    pha = None

    if user_id != None:
        try:
            account = Account.objects.get(email=user_id)
        except Account.DoesNotExist:
            pass

    if pha_email != None:
        try:
            pha = PHA.objects.get(email=pha_email)
        except PHA.DoesNotExist:
            pass

    return account, pha

def fetch_preferences(user_id, pha_email):
    account, pha = resolve_account_pha (user_id, pha_email)
    return Preferences.objects.get_or_create(account=account, pha=pha, defaults={"data": "", "mime": "text/plain"})[0]
<|endoftext|>