<commit_before># -*- coding: utf-8 -*-
'''Provisioning.

Usage:
    exo [options] provision model list [--shared]
    exo [options] provision model info <model>
    exo [options] provision model create (<rid>|<code>) <model_options>...
    exo [options] provision model delete <model>
    exo [options] provision content list <model>
    exo [options] provision content create <model> <id> [<meta>] [--protected]
    exo [options] provision content delete <model> <id>
    exo [options] provision content info <model> <id>
    exo [options] provision content get <model> <id>
    exo [options] provision content put <model> <id>
    exo [options] provision sn list <model> [--offset=num] [--limit=num]
    exo [options] provision sn ranges <model>
    exo [options] provision sn add <model> <sn> [<extra>]
    exo [options] provision sn addcsv <model> <file>
    exo [options] provision sn addrange <model> <format> <length> <casing> <first> <last>
    exo [options] provision sn del <model> <sn>
    exo [options] provision sn delcsv <model> <file>
    exo [options] provision sn delrange <model> <format> <length> <casing> <first> <last>
    exo [options] provision sn rids <model> <sn>
    exo [options] provision sn groups <model> <sn>
    exo [options] provision sn log <model> <sn>
    exo [options] provision sn create <model> <sn> <ownerRID>
    exo [options] provision sn remap <model> <new_sn> <old_sn>
    exo [options] provision sn regenerateCIK <model> <sn>
    exo [options] provision sn disable <model> <sn>

Command Options:

'''
from __future__ import unicode_literals
import inspect
import os
import sys
import re
from pyonep import provision

class Plugin():
    def command(self):
        return 'provision'

    class model:
        def list(self, cmd, args, options):
            pop = options['pop']
            exoconfig = options['config']

            key = exoconfig.config['vendortoken']
            mlist = pop.model_list(key)
            models = mlist.body
            print(models)

        def info(self, cmd, args, options):
            pass
        def create(self, cmd, args, options):
            pass
        def delete(self, cmd, args, options):
            pass

    class content:
        pass
    class sn:
        pass

    def run(self, cmd, args, options):
        cik = options['cik']
        rpc = options['rpc']
        ExoException = options['exception']
        ExoUtilities = options['utils']
        exoconfig = options['config']

        options['pop'] = provision.Provision(manage_by_cik=False,
                                        port='443',
                                        verbose=True,
                                        https=True,
                                        raise_api_exceptions=True)

        found = False
        for name, obj in inspect.getmembers(self, inspect.isclass):
            if name == args['<args>'][0]:
                #print(name)
                for mame, mobj in inspect.getmembers(obj, inspect.ismethod):
                    if mame == args['<args>'][1]:
                        #print(mame)
                        mobj(obj(), mame, args, options)
                        found=True
                        break
                break

        if not found:
            print("not found.")

#  vim: set ai et sw=4 ts=4 :
<commit_msg>Generic recursive class digger.<commit_after># -*- coding: utf-8 -*-
'''Provisioning.

Usage:
    exo [options] provision model list [--shared]
    exo [options] provision model info <model>
    exo [options] provision model create (<rid>|<code>) <model_options>...
    exo [options] provision model delete <model>
    exo [options] provision content list <model>
    exo [options] provision content create <model> <id> [<meta>] [--protected]
    exo [options] provision content delete <model> <id>
    exo [options] provision content info <model> <id>
    exo [options] provision content get <model> <id>
    exo [options] provision content put <model> <id>
    exo [options] provision sn list <model> [--offset=num] [--limit=num]
    exo [options] provision sn ranges <model>
    exo [options] provision sn add <model> <sn> [<extra>]
    exo [options] provision sn addcsv <model> <file>
    exo [options] provision sn addrange <model> <format> <length> <casing> <first> <last>
    exo [options] provision sn del <model> <sn>
    exo [options] provision sn delcsv <model> <file>
    exo [options] provision sn delrange <model> <format> <length> <casing> <first> <last>
    exo [options] provision sn rids <model> <sn>
    exo [options] provision sn groups <model> <sn>
    exo [options] provision sn log <model> <sn>
    exo [options] provision sn create <model> <sn> <ownerRID>
    exo [options] provision sn remap <model> <new_sn> <old_sn>
    exo [options] provision sn regenerateCIK <model> <sn>
    exo [options] provision sn disable <model> <sn>

Command Options:

'''
from __future__ import unicode_literals
import inspect
import os
import sys
import re
from pyonep import provision

class Plugin():
    def command(self):
        return 'provision'

    class model:
        def list(self, cmd, args, options):
            pop = options['pop']
            exoconfig = options['config']

            key = exoconfig.config['vendortoken']
            mlist = pop.model_list(key)
            models = mlist.body
            print(models)

        def info(self, cmd, args, options):
            pass
        def create(self, cmd, args, options):
            pass
        def delete(self, cmd, args, options):
            pass

    class content:
        pass
    class sn:
        pass

    def digMethod(self, arglist, robj):
        for name, obj in inspect.getmembers(robj):
            #print('=', name)
            if name == arglist[0]:
                if inspect.isclass(obj):
                    return self.digMethod(arglist[1:], obj)
                elif inspect.ismethod(obj):
                    return (obj, robj, name)
                break
        return ()

    def run(self, cmd, args, options):
        cik = options['cik']
        rpc = options['rpc']
        ExoException = options['exception']
        ExoUtilities = options['utils']
        exoconfig = options['config']

        options['pop'] = provision.Provision(manage_by_cik=False,
                                        port='443',
                                        verbose=True,
                                        https=True,
                                        raise_api_exceptions=True)

        meth, obj, name = self.digMethod(args['<args>'], self)
        if meth is not None and obj is not None:
            meth(obj(), name, args, options)
        else:
            print("not found")

#  vim: set ai et sw=4 ts=4 :
<|endoftext|>


<commit_before>import copy

from django.db.models.fields.related import ForeignKey, OneToOneField
from rest_framework import mixins
from rest_framework.generics import (
    GenericAPIView, ListAPIView, ListCreateAPIView, RetrieveAPIView,
    RetrieveUpdateDestroyAPIView
)

from api.generics.serializers import (
    DynamicFieldsModelSerializer, DynamicFieldsSerializer
)

class DynamicView(GenericAPIView):

    # foreign / one-to-one fields that can be used with select_related()
    select_related_fields = []
    serializer_fields = []
    field_source_mapping = {}
    fields = ()
    selectable_fields = ()

    def __init__(self, *args, **kwargs):
        """
        Extract prefetches and default fields from Meta
        """
        # TODO: move this to a meta class, to evaluate once when defining the
        # class
        # TODO: This is not efficient - 2016-01-20

        serializer_class = self.get_serializer_class()
        serializer = serializer_class()  # need an instance to extract fields
        model = serializer_class.Meta.model

        assert issubclass(
            serializer_class, DynamicFieldsModelSerializer
        ) or issubclass(serializer_class, DynamicFieldsSerializer), (
            "serializer class must be an instance of \
             DynamicFieldsModelSerializer " "instead got %s"
        ) % (serializer_class.__name__,)

        self.serializer_fields = serializer.fields.keys()

        self.select_related_fields = [
            field.name for field in model._meta.fields
            if isinstance(field, (ForeignKey, OneToOneField))
        ]

        self.field_source_mapping = {
            field.field_name: field.source
            for field in serializer.fields.values()
            if isinstance(
                field, (ForeignKey, OneToOneField)
            )
        }

    def _get_query_fields(self):
        if not self.request:
            return ()

        request_fields = self.request.query_params.get('fields')

        # if requested query fields is set to `all` we will return all
        # serializer fields defined in serializer class. Here we assign
        # `self.fields = ()` so that it will be assigned all serializer
        # fields in `filter_queryset` method.
        if request_fields and request_fields == 'all':
            self.fields = ()
            self.selectable_fields = (self.selectable_fields + tuple(
                self.serializer_fields))
        elif request_fields:
            for request_field in request_fields.split(','):
                if request_field not in list(self.fields):
                    # put selectable fields together with required fields
                    # defined in the class
                    self.fields = self.fields + (request_field,)
                    # just in case if you want to know which of fields
                    # we get as selectable field
                    self.selectable_fields = self.selectable_fields+(request_field,)  # NOQA: E501

            # Some bugs if request fields has 'aggregations'
            # So we need to remove it from request fields.
            # And assign a tuple fields without aggregations
            fields = list(self.fields)
            try:
                fields.remove('aggregations')
            except ValueError:
                pass
            # Assign it again
            self.fields = tuple(fields)

        return getattr(self, 'fields', ())

    def filter_queryset(self, queryset, *args, **kwargs):
        """
        Prefetches based on 'fields' GET arg
        """
        filter_fields = copy.deepcopy(self.request.query_params)

        if 'fields' in filter_fields:
            filter_fields.pop('fields')
        if 'format' in filter_fields:
            filter_fields.pop('format')
        if 'page' in filter_fields:
            filter_fields.pop('page')
        if 'page_size' in filter_fields:
            filter_fields.pop('page_size')
        if 'ordering' in filter_fields:
            filter_fields.pop('ordering')

        for filter_field in filter_fields:
            found = False
            try:
                declared_filters = self.filter_class.declared_filters

                for key in declared_filters:
                    if filter_field == key:
                        found = True
                if found is False:
                    # make error in the code to fail
                    # if input wrong filter name.
                    setattr(self, 'filter_class', 'No Filter Class')
                    break
            except AttributeError:
                pass
        fields = self._get_query_fields(*args, **kwargs)
        if not fields:
            fields = self.serializer_fields

        select_related_fields = list(set(
            self.select_related_fields
        ) & set(fields))

        if select_related_fields:
            queryset = queryset.select_related(*select_related_fields)

        for field in fields:
            # TODO: Hook this up in the view - 2016-01-15
            if hasattr(queryset, 'prefetch_%s' % field):
                queryset = getattr(queryset, 'prefetch_%s' % field)()

        queryset = super(DynamicView, self).filter_queryset(
            queryset, *args, **kwargs
        )

        return queryset

    def get_serializer(self, *args, **kwargs):
        """
        Apply 'fields' to dynamic fields serializer
        """
        fields = self._get_query_fields()
        kwargs['context'] = self.get_serializer_context()
        return super(DynamicView, self).get_serializer(
            fields=fields, *args, **kwargs
        )

class DynamicListView(DynamicView, ListAPIView):
    """
    List view with dynamic properties
    """

class DynamicDetailView(DynamicView, RetrieveAPIView):
    """
    List view with dynamic properties
    """

class DynamicListCRUDView(DynamicView, ListCreateAPIView):
    """
    List view with dynamic properties
    """

class DynamicDetailCRUDView(DynamicView, RetrieveUpdateDestroyAPIView):
    """
    List view with dynamic properties
    """

class SaveAllSerializer(mixins.RetrieveModelMixin,
                        mixins.UpdateModelMixin,
                        mixins.DestroyModelMixin,
                        mixins.CreateModelMixin,
                        GenericAPIView):

    def get(self, request, *args, **kwargs):
        return self.retrieve(request, *args, **kwargs)

    def post(self, request, *args, **kwargs):
        return self.create(request, *args, **kwargs)

    def put(self, request, *args, **kwargs):
        return self.update(request, *args, **kwargs)

    def patch(self, request, *args, **kwargs):
        return self.partial_update(request, *args, **kwargs)

    def delete(self, request, *args, **kwargs):
        return self.destroy(request, *args, **kwargs)
<commit_msg>fix: internal error when text search<commit_after>import copy

from django.db.models.fields.related import ForeignKey, OneToOneField
from rest_framework import mixins
from rest_framework.generics import (
    GenericAPIView, ListAPIView, ListCreateAPIView, RetrieveAPIView,
    RetrieveUpdateDestroyAPIView
)

from api.generics.serializers import (
    DynamicFieldsModelSerializer, DynamicFieldsSerializer
)

class DynamicView(GenericAPIView):

    # foreign / one-to-one fields that can be used with select_related()
    select_related_fields = []
    serializer_fields = []
    field_source_mapping = {}
    fields = ()
    selectable_fields = ()

    def __init__(self, *args, **kwargs):
        """
        Extract prefetches and default fields from Meta
        """
        # TODO: move this to a meta class, to evaluate once when defining the
        # class
        # TODO: This is not efficient - 2016-01-20

        serializer_class = self.get_serializer_class()
        serializer = serializer_class()  # need an instance to extract fields
        model = serializer_class.Meta.model

        assert issubclass(
            serializer_class, DynamicFieldsModelSerializer
        ) or issubclass(serializer_class, DynamicFieldsSerializer), (
            "serializer class must be an instance of \
             DynamicFieldsModelSerializer " "instead got %s"
        ) % (serializer_class.__name__,)

        self.serializer_fields = serializer.fields.keys()

        self.select_related_fields = [
            field.name for field in model._meta.fields
            if isinstance(field, (ForeignKey, OneToOneField))
        ]

        self.field_source_mapping = {
            field.field_name: field.source
            for field in serializer.fields.values()
            if isinstance(
                field, (ForeignKey, OneToOneField)
            )
        }

    def _get_query_fields(self):
        if not self.request:
            return ()

        request_fields = self.request.query_params.get('fields')

        # if requested query fields is set to `all` we will return all
        # serializer fields defined in serializer class. Here we assign
        # `self.fields = ()` so that it will be assigned all serializer
        # fields in `filter_queryset` method.
        if request_fields and request_fields == 'all':
            self.fields = ()
            self.selectable_fields = (self.selectable_fields + tuple(
                self.serializer_fields))
        elif request_fields:
            for request_field in request_fields.split(','):
                if request_field not in list(self.fields):
                    # put selectable fields together with required fields
                    # defined in the class
                    self.fields = self.fields + (request_field,)
                    # just in case if you want to know which of fields
                    # we get as selectable field
                    self.selectable_fields = self.selectable_fields+(request_field,)  # NOQA: E501

            # Some bugs if request fields has 'aggregations'
            # So we need to remove it from request fields.
            # And assign a tuple fields without aggregations
            fields = list(self.fields)
            try:
                fields.remove('aggregations')
            except ValueError:
                pass
            # Assign it again
            self.fields = tuple(fields)

        return getattr(self, 'fields', ())

    def filter_queryset(self, queryset, *args, **kwargs):
        """
        Prefetches based on 'fields' GET arg
        """
        filter_fields = copy.deepcopy(self.request.query_params)

        if 'fields' in filter_fields:
            filter_fields.pop('fields')
        if 'format' in filter_fields:
            filter_fields.pop('format')
        if 'page' in filter_fields:
            filter_fields.pop('page')
        if 'page_size' in filter_fields:
            filter_fields.pop('page_size')
        if 'ordering' in filter_fields:
            filter_fields.pop('ordering')
        if 'q'in filter_fields:
            filter_fields.pop('q')
        if 'q_fields' in filter_fields:
            filter_fields.pop('q_fields')

        for filter_field in filter_fields:
            found = False
            try:
                declared_filters = self.filter_class.declared_filters

                for key in declared_filters:
                    if filter_field == key:
                        found = True
                if found is False:
                    # make error in the code to fail
                    # if input wrong filter name.
                    setattr(self, 'filter_class', 'No Filter Class')
                    break
            except AttributeError:
                pass
        fields = self._get_query_fields(*args, **kwargs)
        if not fields:
            fields = self.serializer_fields

        select_related_fields = list(set(
            self.select_related_fields
        ) & set(fields))

        if select_related_fields:
            queryset = queryset.select_related(*select_related_fields)

        for field in fields:
            # TODO: Hook this up in the view - 2016-01-15
            if hasattr(queryset, 'prefetch_%s' % field):
                queryset = getattr(queryset, 'prefetch_%s' % field)()

        queryset = super(DynamicView, self).filter_queryset(
            queryset, *args, **kwargs
        )

        return queryset

    def get_serializer(self, *args, **kwargs):
        """
        Apply 'fields' to dynamic fields serializer
        """
        fields = self._get_query_fields()
        kwargs['context'] = self.get_serializer_context()
        return super(DynamicView, self).get_serializer(
            fields=fields, *args, **kwargs
        )

class DynamicListView(DynamicView, ListAPIView):
    """
    List view with dynamic properties
    """

class DynamicDetailView(DynamicView, RetrieveAPIView):
    """
    List view with dynamic properties
    """

class DynamicListCRUDView(DynamicView, ListCreateAPIView):
    """
    List view with dynamic properties
    """

class DynamicDetailCRUDView(DynamicView, RetrieveUpdateDestroyAPIView):
    """
    List view with dynamic properties
    """

class SaveAllSerializer(mixins.RetrieveModelMixin,
                        mixins.UpdateModelMixin,
                        mixins.DestroyModelMixin,
                        mixins.CreateModelMixin,
                        GenericAPIView):

    def get(self, request, *args, **kwargs):
        return self.retrieve(request, *args, **kwargs)

    def post(self, request, *args, **kwargs):
        return self.create(request, *args, **kwargs)

    def put(self, request, *args, **kwargs):
        return self.update(request, *args, **kwargs)

    def patch(self, request, *args, **kwargs):
        return self.partial_update(request, *args, **kwargs)

    def delete(self, request, *args, **kwargs):
        return self.destroy(request, *args, **kwargs)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#  * Neither the name of NVIDIA CORPORATION nor the names of its
#    contributors may be used to endorse or promote products derived
#    from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import argparse
import numpy as np
from PIL import Image
import sys

import tritongrpcclient.core as grpcclient
import tritongrpcclient.model_config_pb2 as mc
import tritonhttpclient.core as httpclient
from tritonhttpclient.utils import triton_to_np_dtype
from tritonhttpclient.utils import InferenceServerException

FLAGS = None

def parse_model_grpc(model_metadata, model_config):
    """
    Check the configuration of a model to make sure it meets the
    requirements for an image classification network (as expected by
    this client)
    """
    if len(model_metadata.inputs) != 1:
        raise Exception("expecting 1 input, got {}".format(
            len(model_metadata.inputs)))
    if len(model_metadata.outputs) != 1:
        raise Exception("expecting 1 output, got {}".format(
            len(model_metadata.outputs)))

    if len(model_config.input) != 1:
        raise Exception(
            "expecting 1 input in model configuration, got {}".format(
                len(model_config.input)))

    input_metadata = model_metadata.inputs[0]
    input_config = model_config.input[0]
    output_metadata = model_metadata.outputs[0]

    if output_metadata.datatype != "FP32":
        raise Exception("expecting output datatype to be FP32, model '" +
                        model_metadata.name + "' output type is " +
                        output_metadata.datatype)

    # Output is expected to be a vector. But allow any number of
    # dimensions as long as all but 1 is size 1 (e.g. { 10 }, { 1, 10
    # }, { 10, 1, 1 } are all ok).
    non_one_cnt = 0
    for dim in output_metadata.shape:
        if dim > 1:
            non_one_cnt += 1
            if non_one_cnt > 1:
                raise Exception("expecting model output to be a vector")

    # Model input must have 3 dims, either CHW or HWC
    if len(input_metadata.shape) != 3:
        raise Exception(
            "expecting input to have 3 dimensions, model '{}' input has {}".
            format(model_name, len(input_metadata.shape)))

    if ((input_config.format != mc.ModelInput.FORMAT_NCHW) and
            (input_config.format != mc.ModelInput.FORMAT_NHWC)):
        raise Exception("unexpected input format " +
                        mc.ModelInput.Format.Name(input_config.format) +
                        ", expecting " +
                        mc.ModelInput.Format.Name(mc.ModelInput.FORMAT_NCHW) +
                        " or " +
                        mc.ModelInput.Format.Name(mc.ModelInput.FORMAT_NHWC))

    if input_config.format == mc.ModelInput.FORMAT_NHWC:
        h = input_metadata.shape[0]
        w = input_metadata.shape[1]
        c = input_metadata.shape[2]
    else:
        c = input_metadata.shape[0]
        h = input_metadata.shape[1]
        w = input_metadata.shape[2]

    return (input_metadata.name, output_metadata.name, c, h, w,
            input_config.format, input_metadata.datatype)

def parse_model_http(model_metadata, model_config):
    """
    Check the configuration of a model to make sure it meets the
    requirements for an image classification network (as expected by
    this client)
    """
    if len(model_metadata['inputs']) != 1:
        raise Exception("expecting 1 input, got {}".format(
            len(model_metadata['inputs'])))
    if len(model_metadata['outputs']) != 1:
        raise Exception("expecting 1 output, got {}".format(
            len(model_metadata['outputs'])))

    if len(model_config['input']) != 1:
        raise Exception(
            "expecting 1 input in model configuration, got {}".format(
                len(model_config['input'])))

    input_metadata = model_metadata['inputs'][0]
    input_config = model_config['input'][0]
    output_metadata = model_metadata['outputs'][0]

    if output_metadata['datatype'] != "FP32":
        raise Exception("expecting output datatype to be FP32, model '" +
                        model_metadata['name'] + "' output type is " +
                        output_metadata['datatype'])

    # Output is expected to be a vector. But allow any number of
    # dimensions as long as all but 1 is size 1 (e.g. { 10 }, { 1, 10
    # }, { 10, 1, 1 } are all ok).
    non_one_cnt = 0
    for dim in output_metadata['shape']:
        if dim > 1:
            non_one_cnt += 1
            if non_one_cnt > 1:
                raise Exception("expecting model output to be a vector")

    # Model input must have 3 dims, either CHW or HWC
    if len(input_metadata['shape']) != 3:
        raise Exception(
            "expecting input to have 3 dimensions, model '{}' input has {}".
            format(model_metadata.name, len(input_metadata['shape'])))

    if ((input_config['format'] != "FORMAT_NCHW") and
            (input_config['format'] != "FORMAT_NHWC")):
        raise Exception("unexpected input format " + input_config['format']
                        + ", expecting FORMAT_NCHW or FORMAT_NHWC")

    if input_config['format'] == "FORMAT_NHWC":
        h = input_metadata['shape'][0]
        w = input_metadata['shape'][1]
        c = input_metadata['shape'][2]
    else:
        c = input_metadata['shape'][0]
        h = input_metadata['shape'][1]
        w = input_metadata['shape'][2]

    return (input_metadata['name'], output_metadata['name'], c, h, w,
            "FORMAT_NHWC", input_metadata['datatype'])

def preprocess(img, format, dtype, c, h, w, scaling):
    """
    Pre-process an image to meet the size, type and format
    requirements specified by the parameters.
    """
    # np.set_printoptions(threshold='nan')

    if c == 1:
        sample_img = img.convert('L')
    else:
        sample_img = img.convert('RGB')

    resized_img = sample_img.resize((w, h), Image.BILINEAR)
    resized = np.array(resized_img)
    if resized.ndim == 2:
        resized = resized[:, :, np.newaxis]

    npdtype = triton_to_np_dtype(dtype)
    typed = resized.astype(npdtype)

    if scaling == 'INCEPTION':
        scaled = (typed / 128) - 1
    elif scaling == 'VGG':
        if c == 1:
            scaled = typed - np.asarray((128,), dtype=npdtype)
        else:
            scaled = typed - np.asarray((123, 117, 104), dtype=npdtype)
    else:
        scaled = typed

    # Swap to CHW if necessary
    if FLAGS.protocol.lower() == "grpc":
        if format == mc.ModelInput.FORMAT_NCHW:
            ordered = np.transpose(scaled, (2, 0, 1))
        else:
            ordered = scaled
    else:
        if format == "FORMAT_NCHW":
            ordered = np.transpose(scaled, (2, 0, 1))
        else:
            ordered = scaled

    # Channels are in RGB order. Currently model configuration data
    # doesn't provide any information as to other channel orderings
    # (like BGR) so we just assume RGB.
    return ordered

def postprocess(results, output_name, batch_size):
    """
    Post-process results to show classifications.
    """

    output_array = results.as_numpy(output_name)
    if len(output_array) != batch_size:
        raise Exception("expected {} results, got {}".format(
            batch_size, len(output_array)))

    for results in output_array:
        for result in results:
            if output_array.dtype.type == np.bytes_:
                cls = "".join(chr(x) for x in result).split(':')
            else:
                cls = result.split(':')
            print("    {} ({}) = {}".format(cls[0], cls[1], cls[2]))

def requestGenerator(input_name, output_name, c, h, w, format, dtype, FLAGS):
    # Preprocess image into input data according to model requirements
    image_data = None
    with Image.open(FLAGS.image_filename) as img:
        image_data = preprocess(img, format, dtype, c, h, w, FLAGS.scaling)

    repeated_image_data = [image_data for _ in range(FLAGS.batch_size)]
    batched_image_data = np.stack(repeated_image_data, axis=0)

    # Set the input data
    inputs = []
    if FLAGS.protocol.lower() == "grpc":
        inputs.append(grpcclient.InferInput(input_name, batched_image_data.shape, dtype))
        inputs[0].set_data_from_numpy(batched_image_data)
    else:
        inputs.append(httpclient.InferInput(input_name, batched_image_data.shape, dtype))
        inputs[0].set_data_from_numpy(batched_image_data, binary_data=False)

    outputs = []
    if FLAGS.protocol.lower() == "grpc":
        outputs.append(grpcclient.InferRequestedOutput(output_name, class_count=FLAGS.classes))
    else:
        outputs.append(httpclient.InferRequestedOutput(
            output_name, binary_data=False, class_count=FLAGS.classes))

    yield inputs, outputs, FLAGS.model_name, FLAGS.model_version

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-v',
                        '--verbose',
                        action="store_true",
                        required=False,
                        default=False,
                        help='Enable verbose output')
    parser.add_argument('-m',
                        '--model-name',
                        type=str,
                        required=True,
                        help='Name of model')
    parser.add_argument('-x',
                        '--model-version',
                        type=str,
                        required=False,
                        default="",
                        help='Version of model. Default is to use latest version.')
    parser.add_argument('-b',
                        '--batch-size',
                        type=int,
                        required=False,
                        default=1,
                        help='Batch size. Default is 1.')
    parser.add_argument('-c',
                        '--classes',
                        type=int,
                        required=False,
                        default=1,
                        help='Number of class results to report. Default is 1.')
    parser.add_argument('-s',
                        '--scaling',
                        type=str,
                        choices=['NONE', 'INCEPTION', 'VGG'],
                        required=False,
                        default='NONE',
                        help='Type of scaling to apply to image pixels. Default is NONE.')
    parser.add_argument('-u',
                        '--url',
                        type=str,
                        required=False,
                        default='localhost:8000',
                        help='Inference server URL. Default is localhost:8000.')
    parser.add_argument('-i',
                        '--protocol',
                        type=str,
                        required=False,
                        default='HTTP',
                        help='Protocol (HTTP/gRPC) used to communicate with ' +
                        'the inference service. Default is HTTP.')
    parser.add_argument('image_filename', type=str, help='Input image.')
    FLAGS = parser.parse_args()

    try:
        if FLAGS.protocol.lower() == "grpc":
            # Create gRPC client for communicating with the server
            triton_client = grpcclient.InferenceServerClient(FLAGS.url)
        else:
            # Create HTTP client for communicating with the server
            triton_client = httpclient.InferenceServerClient(FLAGS.url)
    except Exception as e:
        print("context creation failed: " + str(e))
        sys.exit()

    # Make sure the model matches our requirements, and get some
    # properties of the model that we need for preprocessing
    try:
        model_metadata = triton_client.get_model_metadata(
            model_name=FLAGS.model_name, model_version=FLAGS.model_version)
    except InferenceServerException as e:
        print("failed to retrieve the metadata: " + str(e))
        sys.exit()

    try:
        model_config = triton_client.get_model_config(
            model_name=FLAGS.model_name, model_version=FLAGS.model_version)
    except InferenceServerException as e:
        print("failed to retrieve the config: " + str(e))
        sys.exit()

    if FLAGS.protocol.lower() == "grpc":
        input_name, output_name, c, h, w, format, dtype = parse_model_grpc(
            model_metadata, model_config.config)
    else:
        input_name, output_name, c, h, w, format, dtype = parse_model_http(
            model_metadata, model_config)

    # Send requests of FLAGS.batch_size images. If the number of
    # images isn't an exact multiple of FLAGS.batch_size then just
    # start over with the first images until the batch is filled.
    requests = []
    responses = []

    # Send request
    try:
        for inputs, outputs, model_name, model_version in requestGenerator(
                input_name, output_name, c, h, w, format, dtype, FLAGS):
            responses.append(
                triton_client.infer(FLAGS.model_name,
                                    inputs,
                                    model_version=FLAGS.model_version,
                                    outputs=outputs))

    except InferenceServerException as e:
        print("inference failed: " + str(e))
        sys.exit()

    for response in responses:
        postprocess(response, output_name, FLAGS.batch_size)

    print("PASS")
<commit_msg>Fix typo during http model config parsing (#1342)<commit_after>#!/usr/bin/env python
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#  * Neither the name of NVIDIA CORPORATION nor the names of its
#    contributors may be used to endorse or promote products derived
#    from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import argparse
import numpy as np
from PIL import Image
import sys

import tritongrpcclient.core as grpcclient
import tritongrpcclient.model_config_pb2 as mc
import tritonhttpclient.core as httpclient
from tritonhttpclient.utils import triton_to_np_dtype
from tritonhttpclient.utils import InferenceServerException

FLAGS = None

def parse_model_grpc(model_metadata, model_config):
    """
    Check the configuration of a model to make sure it meets the
    requirements for an image classification network (as expected by
    this client)
    """
    if len(model_metadata.inputs) != 1:
        raise Exception("expecting 1 input, got {}".format(
            len(model_metadata.inputs)))
    if len(model_metadata.outputs) != 1:
        raise Exception("expecting 1 output, got {}".format(
            len(model_metadata.outputs)))

    if len(model_config.input) != 1:
        raise Exception(
            "expecting 1 input in model configuration, got {}".format(
                len(model_config.input)))

    input_metadata = model_metadata.inputs[0]
    input_config = model_config.input[0]
    output_metadata = model_metadata.outputs[0]

    if output_metadata.datatype != "FP32":
        raise Exception("expecting output datatype to be FP32, model '" +
                        model_metadata.name + "' output type is " +
                        output_metadata.datatype)

    # Output is expected to be a vector. But allow any number of
    # dimensions as long as all but 1 is size 1 (e.g. { 10 }, { 1, 10
    # }, { 10, 1, 1 } are all ok).
    non_one_cnt = 0
    for dim in output_metadata.shape:
        if dim > 1:
            non_one_cnt += 1
            if non_one_cnt > 1:
                raise Exception("expecting model output to be a vector")

    # Model input must have 3 dims, either CHW or HWC
    if len(input_metadata.shape) != 3:
        raise Exception(
            "expecting input to have 3 dimensions, model '{}' input has {}".
            format(model_name, len(input_metadata.shape)))

    if ((input_config.format != mc.ModelInput.FORMAT_NCHW) and
            (input_config.format != mc.ModelInput.FORMAT_NHWC)):
        raise Exception("unexpected input format " +
                        mc.ModelInput.Format.Name(input_config.format) +
                        ", expecting " +
                        mc.ModelInput.Format.Name(mc.ModelInput.FORMAT_NCHW) +
                        " or " +
                        mc.ModelInput.Format.Name(mc.ModelInput.FORMAT_NHWC))

    if input_config.format == mc.ModelInput.FORMAT_NHWC:
        h = input_metadata.shape[0]
        w = input_metadata.shape[1]
        c = input_metadata.shape[2]
    else:
        c = input_metadata.shape[0]
        h = input_metadata.shape[1]
        w = input_metadata.shape[2]

    return (input_metadata.name, output_metadata.name, c, h, w,
            input_config.format, input_metadata.datatype)

def parse_model_http(model_metadata, model_config):
    """
    Check the configuration of a model to make sure it meets the
    requirements for an image classification network (as expected by
    this client)
    """
    if len(model_metadata['inputs']) != 1:
        raise Exception("expecting 1 input, got {}".format(
            len(model_metadata['inputs'])))
    if len(model_metadata['outputs']) != 1:
        raise Exception("expecting 1 output, got {}".format(
            len(model_metadata['outputs'])))

    if len(model_config['input']) != 1:
        raise Exception(
            "expecting 1 input in model configuration, got {}".format(
                len(model_config['input'])))

    input_metadata = model_metadata['inputs'][0]
    input_config = model_config['input'][0]
    output_metadata = model_metadata['outputs'][0]

    if output_metadata['datatype'] != "FP32":
        raise Exception("expecting output datatype to be FP32, model '" +
                        model_metadata['name'] + "' output type is " +
                        output_metadata['datatype'])

    # Output is expected to be a vector. But allow any number of
    # dimensions as long as all but 1 is size 1 (e.g. { 10 }, { 1, 10
    # }, { 10, 1, 1 } are all ok).
    non_one_cnt = 0
    for dim in output_metadata['shape']:
        if dim > 1:
            non_one_cnt += 1
            if non_one_cnt > 1:
                raise Exception("expecting model output to be a vector")

    # Model input must have 3 dims, either CHW or HWC
    if len(input_metadata['shape']) != 3:
        raise Exception(
            "expecting input to have 3 dimensions, model '{}' input has {}".
            format(model_metadata.name, len(input_metadata['shape'])))

    if ((input_config['format'] != "FORMAT_NCHW") and
            (input_config['format'] != "FORMAT_NHWC")):
        raise Exception("unexpected input format " + input_config['format']
                        + ", expecting FORMAT_NCHW or FORMAT_NHWC")

    if input_config['format'] == "FORMAT_NHWC":
        h = input_metadata['shape'][0]
        w = input_metadata['shape'][1]
        c = input_metadata['shape'][2]
    else:
        c = input_metadata['shape'][0]
        h = input_metadata['shape'][1]
        w = input_metadata['shape'][2]

    return (input_metadata['name'], output_metadata['name'], c, h, w,
            input_config['format'], input_metadata['datatype'])

def preprocess(img, format, dtype, c, h, w, scaling):
    """
    Pre-process an image to meet the size, type and format
    requirements specified by the parameters.
    """
    # np.set_printoptions(threshold='nan')

    if c == 1:
        sample_img = img.convert('L')
    else:
        sample_img = img.convert('RGB')

    resized_img = sample_img.resize((w, h), Image.BILINEAR)
    resized = np.array(resized_img)
    if resized.ndim == 2:
        resized = resized[:, :, np.newaxis]

    npdtype = triton_to_np_dtype(dtype)
    typed = resized.astype(npdtype)

    if scaling == 'INCEPTION':
        scaled = (typed / 128) - 1
    elif scaling == 'VGG':
        if c == 1:
            scaled = typed - np.asarray((128,), dtype=npdtype)
        else:
            scaled = typed - np.asarray((123, 117, 104), dtype=npdtype)
    else:
        scaled = typed

    # Swap to CHW if necessary
    if FLAGS.protocol.lower() == "grpc":
        if format == mc.ModelInput.FORMAT_NCHW:
            ordered = np.transpose(scaled, (2, 0, 1))
        else:
            ordered = scaled
    else:
        if format == "FORMAT_NCHW":
            ordered = np.transpose(scaled, (2, 0, 1))
        else:
            ordered = scaled

    # Channels are in RGB order. Currently model configuration data
    # doesn't provide any information as to other channel orderings
    # (like BGR) so we just assume RGB.
    return ordered

def postprocess(results, output_name, batch_size):
    """
    Post-process results to show classifications.
    """

    output_array = results.as_numpy(output_name)
    if len(output_array) != batch_size:
        raise Exception("expected {} results, got {}".format(
            batch_size, len(output_array)))

    for results in output_array:
        for result in results:
            if output_array.dtype.type == np.bytes_:
                cls = "".join(chr(x) for x in result).split(':')
            else:
                cls = result.split(':')
            print("    {} ({}) = {}".format(cls[0], cls[1], cls[2]))

def requestGenerator(input_name, output_name, c, h, w, format, dtype, FLAGS):
    # Preprocess image into input data according to model requirements
    image_data = None
    with Image.open(FLAGS.image_filename) as img:
        image_data = preprocess(img, format, dtype, c, h, w, FLAGS.scaling)

    repeated_image_data = [image_data for _ in range(FLAGS.batch_size)]
    batched_image_data = np.stack(repeated_image_data, axis=0)

    # Set the input data
    inputs = []
    if FLAGS.protocol.lower() == "grpc":
        inputs.append(grpcclient.InferInput(input_name, batched_image_data.shape, dtype))
        inputs[0].set_data_from_numpy(batched_image_data)
    else:
        inputs.append(httpclient.InferInput(input_name, batched_image_data.shape, dtype))
        inputs[0].set_data_from_numpy(batched_image_data, binary_data=False)

    outputs = []
    if FLAGS.protocol.lower() == "grpc":
        outputs.append(grpcclient.InferRequestedOutput(output_name, class_count=FLAGS.classes))
    else:
        outputs.append(httpclient.InferRequestedOutput(
            output_name, binary_data=False, class_count=FLAGS.classes))

    yield inputs, outputs, FLAGS.model_name, FLAGS.model_version

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-v',
                        '--verbose',
                        action="store_true",
                        required=False,
                        default=False,
                        help='Enable verbose output')
    parser.add_argument('-m',
                        '--model-name',
                        type=str,
                        required=True,
                        help='Name of model')
    parser.add_argument('-x',
                        '--model-version',
                        type=str,
                        required=False,
                        default="",
                        help='Version of model. Default is to use latest version.')
    parser.add_argument('-b',
                        '--batch-size',
                        type=int,
                        required=False,
                        default=1,
                        help='Batch size. Default is 1.')
    parser.add_argument('-c',
                        '--classes',
                        type=int,
                        required=False,
                        default=1,
                        help='Number of class results to report. Default is 1.')
    parser.add_argument('-s',
                        '--scaling',
                        type=str,
                        choices=['NONE', 'INCEPTION', 'VGG'],
                        required=False,
                        default='NONE',
                        help='Type of scaling to apply to image pixels. Default is NONE.')
    parser.add_argument('-u',
                        '--url',
                        type=str,
                        required=False,
                        default='localhost:8000',
                        help='Inference server URL. Default is localhost:8000.')
    parser.add_argument('-i',
                        '--protocol',
                        type=str,
                        required=False,
                        default='HTTP',
                        help='Protocol (HTTP/gRPC) used to communicate with ' +
                        'the inference service. Default is HTTP.')
    parser.add_argument('image_filename', type=str, help='Input image.')
    FLAGS = parser.parse_args()

    try:
        if FLAGS.protocol.lower() == "grpc":
            # Create gRPC client for communicating with the server
            triton_client = grpcclient.InferenceServerClient(FLAGS.url)
        else:
            # Create HTTP client for communicating with the server
            triton_client = httpclient.InferenceServerClient(FLAGS.url)
    except Exception as e:
        print("context creation failed: " + str(e))
        sys.exit()

    # Make sure the model matches our requirements, and get some
    # properties of the model that we need for preprocessing
    try:
        model_metadata = triton_client.get_model_metadata(
            model_name=FLAGS.model_name, model_version=FLAGS.model_version)
    except InferenceServerException as e:
        print("failed to retrieve the metadata: " + str(e))
        sys.exit()

    try:
        model_config = triton_client.get_model_config(
            model_name=FLAGS.model_name, model_version=FLAGS.model_version)
    except InferenceServerException as e:
        print("failed to retrieve the config: " + str(e))
        sys.exit()

    if FLAGS.protocol.lower() == "grpc":
        input_name, output_name, c, h, w, format, dtype = parse_model_grpc(
            model_metadata, model_config.config)
    else:
        input_name, output_name, c, h, w, format, dtype = parse_model_http(
            model_metadata, model_config)

    # Send requests of FLAGS.batch_size images. If the number of
    # images isn't an exact multiple of FLAGS.batch_size then just
    # start over with the first images until the batch is filled.
    requests = []
    responses = []

    # Send request
    try:
        for inputs, outputs, model_name, model_version in requestGenerator(
                input_name, output_name, c, h, w, format, dtype, FLAGS):
            responses.append(
                triton_client.infer(FLAGS.model_name,
                                    inputs,
                                    model_version=FLAGS.model_version,
                                    outputs=outputs))

    except InferenceServerException as e:
        print("inference failed: " + str(e))
        sys.exit()

    for response in responses:
        postprocess(response, output_name, FLAGS.batch_size)

    print("PASS")
<|endoftext|>


<commit_before>#!/usr/bin/env python

""" Twitter bot for automatically replying to questions
    asked to @decisiverobot on twitter """

# Stolen shamelessly from:
# github.com/vivekhaldar/the_shrinkbot/blob/master/the_shrink.py
# blog.vivekhaldar.com/post/2830035130/how-to-write-a-twitter-bot-in-python

from twitter.api import Twitter
from twitter.oauth import OAuth

import time
import re

import decisiverobot

# Get your keys from https://dev.twitter.com/apps/new
CONSUMER_KEY = 'replacethis'
CONSUMER_SECRET = 'replacethis'
ACCESS_TOKEN = 'replacethis'
ACCESS_TOKEN_SECRET = 'replacethis'
TWITTER_USER = "@replacethis"

REG_REMOVE_USER = re.compile(r'^' + TWITTER_USER, re.IGNORECASE)
SLEEP_INTERVAL = 30
LAST_ID_FILE = 'twitterbot.lastid'

def run():
    """ Runs the bot it a loop, checking for questions and replying """

    # We use two twitter clients, one to search, another to update. Just
    # easier that way...
    twitter = Twitter(domain='search.twitter.com')
    twitter.uriparts = ()

    last_id_replied = ''

    try:
        with file(LAST_ID_FILE, 'r') as content:
            last_id_replied = content.read()
    except IOError as ex:
        print(LAST_ID_FILE + " not found")

    poster = Twitter(
        auth=OAuth(
            ACCESS_TOKEN, ACCESS_TOKEN_SECRET, CONSUMER_KEY, CONSUMER_SECRET),
        secure=True,
        api_version='1',
        domain='api.twitter.com')

    while True:
        results = twitter.search(
            q=TWITTER_USER,
            since_id=last_id_replied)['results']

        for result in results:
            # Remove our twitter name from the question.
            question = REG_REMOVE_USER.sub('', result['text'])
            asker = result['from_user']
            question_id = str(result['id'])
            print(" <<< " + asker + ": " + question)

            bot_response = decisiverobot.snarkyanswer(question)

            msg = '@{0} {1}'.format(asker, bot_response)
            print('====> Resp =' + msg)
            last_id_replied = question_id
            poster.statuses.update(status=msg)

            print('Last id replied to = ' + str(last_id_replied))

            try:
                with file(LAST_ID_FILE, 'w') as writer:
                    writer.write(last_id_replied)
            except IOError as ex:
                print(ex)

        time.sleep(SLEEP_INTERVAL)

if __name__ == '__main__':
    run()<commit_msg>Twitter bot should only answer tweets where its name is at the start<commit_after>#!/usr/bin/env python

""" Twitter bot for automatically replying to questions
    asked to @decisiverobot on twitter """

# Stolen shamelessly from:
# github.com/vivekhaldar/the_shrinkbot/blob/master/the_shrink.py
# blog.vivekhaldar.com/post/2830035130/how-to-write-a-twitter-bot-in-python

from twitter.api import Twitter
from twitter.oauth import OAuth

import time
import re

import decisiverobot

# Get your keys from https://dev.twitter.com/apps/new
CONSUMER_KEY = 'replacethis'
CONSUMER_SECRET = 'replacethis'
ACCESS_TOKEN = 'replacethis'
ACCESS_TOKEN_SECRET = 'replacethis'
TWITTER_USER = "@replacethis"

REG_USER_BEGINNING = re.compile(r'^\.?' + TWITTER_USER, re.IGNORECASE)
SLEEP_INTERVAL = 30
LAST_ID_FILE = 'twitterbot.lastid'

def run():
    """ Runs the bot it a loop, checking for questions and replying """

    # We use two twitter clients, one to search, another to update. Just
    # easier that way...
    twitter = Twitter(domain='search.twitter.com')
    twitter.uriparts = ()

    last_id_replied = ''

    try:
        with file(LAST_ID_FILE, 'r') as content:
            last_id_replied = content.read()
    except IOError as ex:
        print(LAST_ID_FILE + " not found")

    poster = Twitter(
        auth=OAuth(
            ACCESS_TOKEN, ACCESS_TOKEN_SECRET, CONSUMER_KEY, CONSUMER_SECRET),
        secure=True,
        api_version='1',
        domain='api.twitter.com')

    while True:
        results = twitter.search(
            q=TWITTER_USER,
            since_id=last_id_replied)['results']

        for result in results:
            last_id_replied = str(result['id'])
            question = result['text']

            # Only answer tweets with username at the beginning
            if REG_USER_BEGINNING.match(question) is None:
                continue

            # Remove our twitter name from the question.
            question = REG_USER_BEGINNING.sub('', question)
            asker = result['from_user']
            print(" <<< " + asker + ": " + question)

            bot_response = decisiverobot.snarkyanswer(question)

            msg = '@{0} {1}'.format(asker, bot_response)
            print('====> Resp =' + msg)
            poster.statuses.update(status=msg)

            try:
                with file(LAST_ID_FILE, 'w') as writer:
                    writer.write(last_id_replied)
            except IOError as ex:
                print(ex)

        time.sleep(SLEEP_INTERVAL)

if __name__ == '__main__':
    run()<|endoftext|>


<commit_before>import numpy
import scipy.sparse
import gc
from sppy.linalg.GeneralLinearOperator import GeneralLinearOperator
from apgl.util.Parameter import Parameter
import sppy.linalg

class RandomisedSVD(object):
    """
    Compute the randomised SVD using the algorithm on page 9 of Halko et al.,
    Finding Structure with randomness: stochastic algorithms for constructing
    approximate matrix decompositions, 2009.
    """
    def ___init__(self):
        pass

    @staticmethod
    def svd(A, k, p=10, q=2, omega=None):
        """
        Compute the SVD of a sparse or dense matrix A, finding the first k
        singular vectors/values, using exponent q. Returns the left and right singular
        vectors, and the singular values. The resulting matrix can be approximated
        using A ~ U s V.T.

        :param A: A matrix or GeneralLinearOperator

        :param k: The number of singular values and random projections

        :param p: The oversampling parameter

        :param q: The exponent for the projections.

        :param omega: An initial matrix to perform random projections onto with at least k columns
        """
        return sppy.linalg.rsvd(A, k, p, q, omega)

    @staticmethod
    def updateSvd(A, U, s, V, E, k, p):
        """
        Given a matrix A whose approximate SVD is U s V.T, compute the SVD
        of the new matrix A + E, using previous info. A and E are sparse
        matrices. The rank of the approximation is p, and k is an oversampling
        parameter.
        """
        Parameter.checkInt(k, 1, float("inf"))
        Parameter.checkInt(p, 0, float("inf"))

        if isinstance(E, GeneralLinearOperator):
            M = E
        else:
            M = GeneralLinearOperator.asLinearOperator(E)

        N = GeneralLinearOperator.asLinearOperator(A + E)

        n = A.shape[1]
        omega = numpy.random.randn(n, p)

        Y = U*s + M.matmat(V)
        Y = numpy.c_[Y, N.matmat(omega)]

        Q, R = numpy.linalg.qr(Y)
        del omega

        del Y
        del R
        gc.collect()

        B = N.rmatmat(Q).T
        U, s, V = numpy.linalg.svd(B, full_matrices=False)
        del B
        V = V.T
        U = Q.dot(U)

        U = U[:, 0:k]
        s = s[0:k]
        V = V[:, 0:k]

        return U, s, V

        <commit_msg>Allow A and E to be linear operators <commit_after>import numpy
import scipy.sparse
import gc
from sppy.linalg.GeneralLinearOperator import GeneralLinearOperator
from apgl.util.Parameter import Parameter
import sppy.linalg

class RandomisedSVD(object):
    """
    Compute the randomised SVD using the algorithm on page 9 of Halko et al.,
    Finding Structure with randomness: stochastic algorithms for constructing
    approximate matrix decompositions, 2009.
    """
    def ___init__(self):
        pass

    @staticmethod
    def svd(A, k, p=10, q=2, omega=None):
        """
        Compute the SVD of a sparse or dense matrix A, finding the first k
        singular vectors/values, using exponent q. Returns the left and right singular
        vectors, and the singular values. The resulting matrix can be approximated
        using A ~ U s V.T.

        :param A: A matrix or GeneralLinearOperator

        :param k: The number of singular values and random projections

        :param p: The oversampling parameter

        :param q: The exponent for the projections.

        :param omega: An initial matrix to perform random projections onto with at least k columns
        """
        return sppy.linalg.rsvd(A, k, p, q, omega)

    @staticmethod
    def updateSvd(A, U, s, V, E, k, p=10):
        """
        Given a matrix A whose approximate SVD is U s V.T, compute the SVD
        of the new matrix A + E, using previous info. A and E are sparse
        matrices. The rank of the approximation is p, and k is an oversampling
        parameter.
        """
        Parameter.checkInt(k, 1, float("inf"))
        Parameter.checkInt(p, 0, float("inf"))

        if isinstance(A, GeneralLinearOperator):
            L = A
        else:
            L = GeneralLinearOperator.asLinearOperator(A)

        if isinstance(E, GeneralLinearOperator):
            M = E
        else:
            M = GeneralLinearOperator.asLinearOperator(E)

        N = GeneralLinearOperator.asLinearOperatorSum(L, M)

        n = A.shape[1]
        omega = numpy.random.randn(n, p)

        Y = U*s + M.matmat(V)
        Y = numpy.c_[Y, N.matmat(omega)]

        Q, R = numpy.linalg.qr(Y)
        del omega

        del Y
        del R
        gc.collect()

        B = N.rmatmat(Q).T
        U, s, V = numpy.linalg.svd(B, full_matrices=False)
        del B
        V = V.T
        U = Q.dot(U)

        U = U[:, 0:k]
        s = s[0:k]
        V = V[:, 0:k]

        return U, s, V

        <|endoftext|>


<commit_before>from django.db.models.fields.related import ForeignKey, OneToOneField
from rest_framework import mixins
from rest_framework.generics import (
    GenericAPIView, ListAPIView, ListCreateAPIView, RetrieveAPIView,
    RetrieveUpdateDestroyAPIView
)

from api.generics.serializers import (
    DynamicFieldsModelSerializer, DynamicFieldsSerializer
)

class DynamicView(GenericAPIView):

    # foreign / one-to-one fields that can be used with select_related()
    select_related_fields = []
    serializer_fields = []
    field_source_mapping = {}
    fields = ()
    selectable_fields = ()

    def __init__(self, *args, **kwargs):
        """
        Extract prefetches and default fields from Meta
        """
        # TODO: move this to a meta class, to evaluate once when defining the
        # class
        # TODO: This is not efficient - 2016-01-20

        serializer_class = self.get_serializer_class()
        serializer = serializer_class()  # need an instance to extract fields
        model = serializer_class.Meta.model

        assert issubclass(
            serializer_class, DynamicFieldsModelSerializer
        ) or issubclass(serializer_class, DynamicFieldsSerializer), (
            "serializer class must be an instance of \
             DynamicFieldsModelSerializer " "instead got %s"
        ) % (serializer_class.__name__,)

        self.serializer_fields = serializer.fields.keys()

        self.select_related_fields = [
            field.name for field in model._meta.fields
            if isinstance(field, (ForeignKey, OneToOneField))
        ]

        self.field_source_mapping = {
            field.field_name: field.source
            for field in serializer.fields.values()
            if isinstance(
                field, (ForeignKey, OneToOneField)
            )
        }

    def _get_query_fields(self):
        if not self.request:
            return ()

        request_fields = self.request.query_params.get('fields')

        # if requested query fields is set to `all` we will return all
        # serializer fields defined in serializer class. Here we assign
        # `self.fields = ()` so that it will be assigned all serializer
        # fields in `filter_queryset` method.
        if request_fields and request_fields == 'all':
            self.fields = ()
        elif request_fields:
            for request_field in request_fields.split(','):
                if request_field not in list(self.fields):
                    # put selectable fields together with required fields
                    # defined in the class
                    self.fields = self.fields + (request_field,)
                    # just in case if you want to know which of fields
                    # we get as selectable field
                    self.selectable_fields = self.selectable_fields+(request_field,)  # NOQA: E501

            # Some bugs if request fields has 'aggregations'
            # So we need to remove it from request fields.
            # And assign a tuple fields without aggregations
            fields = list(self.fields)
            try:
                fields.remove('aggregations')
            except Exception as e:
                pass
            # Assign it again
            self.fields = tuple(fields)

        return getattr(self, 'fields', ())

    def filter_queryset(self, queryset, *args, **kwargs):
        """
        Prefetches based on 'fields' GET arg
        """

        fields = self._get_query_fields(*args, **kwargs)
        if not fields:
            fields = self.serializer_fields

        select_related_fields = list(set(
            self.select_related_fields
        ) & set(fields))

        if select_related_fields:
            queryset = queryset.select_related(*select_related_fields)

        for field in fields:
            # TODO: Hook this up in the view - 2016-01-15
            if hasattr(queryset, 'prefetch_%s' % field):
                queryset = getattr(queryset, 'prefetch_%s' % field)()

        queryset = super(DynamicView, self).filter_queryset(
            queryset, *args, **kwargs
        )

        return queryset

    def get_serializer(self, *args, **kwargs):
        """
        Apply 'fields' to dynamic fields serializer
        """
        fields = self._get_query_fields()
        kwargs['context'] = self.get_serializer_context()
        return super(DynamicView, self).get_serializer(
            fields=fields, *args, **kwargs
        )

class DynamicListView(DynamicView, ListAPIView):
    """
    List view with dynamic properties
    """

class DynamicDetailView(DynamicView, RetrieveAPIView):
    """
    List view with dynamic properties
    """

class DynamicListCRUDView(DynamicView, ListCreateAPIView):
    """
    List view with dynamic properties
    """

class DynamicDetailCRUDView(DynamicView, RetrieveUpdateDestroyAPIView):
    """
    List view with dynamic properties
    """

class SaveAllSerializer(mixins.RetrieveModelMixin,
                        mixins.UpdateModelMixin,
                        mixins.DestroyModelMixin,
                        mixins.CreateModelMixin,
                        GenericAPIView):

    def get(self, request, *args, **kwargs):
        return self.retrieve(request, *args, **kwargs)

    def post(self, request, *args, **kwargs):
        return self.create(request, *args, **kwargs)

    def put(self, request, *args, **kwargs):
        return self.update(request, *args, **kwargs)

    def patch(self, request, *args, **kwargs):
        return self.partial_update(request, *args, **kwargs)

    def delete(self, request, *args, **kwargs):
        return self.destroy(request, *args, **kwargs)
<commit_msg>Unused code<commit_after>from django.db.models.fields.related import ForeignKey, OneToOneField
from rest_framework import mixins
from rest_framework.generics import (
    GenericAPIView, ListAPIView, ListCreateAPIView, RetrieveAPIView,
    RetrieveUpdateDestroyAPIView
)

from api.generics.serializers import (
    DynamicFieldsModelSerializer, DynamicFieldsSerializer
)

class DynamicView(GenericAPIView):

    # foreign / one-to-one fields that can be used with select_related()
    select_related_fields = []
    serializer_fields = []
    field_source_mapping = {}
    fields = ()
    selectable_fields = ()

    def __init__(self, *args, **kwargs):
        """
        Extract prefetches and default fields from Meta
        """
        # TODO: move this to a meta class, to evaluate once when defining the
        # class
        # TODO: This is not efficient - 2016-01-20

        serializer_class = self.get_serializer_class()
        serializer = serializer_class()  # need an instance to extract fields
        model = serializer_class.Meta.model

        assert issubclass(
            serializer_class, DynamicFieldsModelSerializer
        ) or issubclass(serializer_class, DynamicFieldsSerializer), (
            "serializer class must be an instance of \
             DynamicFieldsModelSerializer " "instead got %s"
        ) % (serializer_class.__name__,)

        self.serializer_fields = serializer.fields.keys()

        self.select_related_fields = [
            field.name for field in model._meta.fields
            if isinstance(field, (ForeignKey, OneToOneField))
        ]

        self.field_source_mapping = {
            field.field_name: field.source
            for field in serializer.fields.values()
            if isinstance(
                field, (ForeignKey, OneToOneField)
            )
        }

    def _get_query_fields(self):
        if not self.request:
            return ()

        request_fields = self.request.query_params.get('fields')

        # if requested query fields is set to `all` we will return all
        # serializer fields defined in serializer class. Here we assign
        # `self.fields = ()` so that it will be assigned all serializer
        # fields in `filter_queryset` method.
        if request_fields and request_fields == 'all':
            self.fields = ()
        elif request_fields:
            for request_field in request_fields.split(','):
                if request_field not in list(self.fields):
                    # put selectable fields together with required fields
                    # defined in the class
                    self.fields = self.fields + (request_field,)
                    # just in case if you want to know which of fields
                    # we get as selectable field
                    self.selectable_fields = self.selectable_fields+(request_field,)  # NOQA: E501

            # Some bugs if request fields has 'aggregations'
            # So we need to remove it from request fields.
            # And assign a tuple fields without aggregations
            fields = list(self.fields)
            try:
                fields.remove('aggregations')
            except Exception:
                pass
            # Assign it again
            self.fields = tuple(fields)

        return getattr(self, 'fields', ())

    def filter_queryset(self, queryset, *args, **kwargs):
        """
        Prefetches based on 'fields' GET arg
        """

        fields = self._get_query_fields(*args, **kwargs)
        if not fields:
            fields = self.serializer_fields

        select_related_fields = list(set(
            self.select_related_fields
        ) & set(fields))

        if select_related_fields:
            queryset = queryset.select_related(*select_related_fields)

        for field in fields:
            # TODO: Hook this up in the view - 2016-01-15
            if hasattr(queryset, 'prefetch_%s' % field):
                queryset = getattr(queryset, 'prefetch_%s' % field)()

        queryset = super(DynamicView, self).filter_queryset(
            queryset, *args, **kwargs
        )

        return queryset

    def get_serializer(self, *args, **kwargs):
        """
        Apply 'fields' to dynamic fields serializer
        """
        fields = self._get_query_fields()
        kwargs['context'] = self.get_serializer_context()
        return super(DynamicView, self).get_serializer(
            fields=fields, *args, **kwargs
        )

class DynamicListView(DynamicView, ListAPIView):
    """
    List view with dynamic properties
    """

class DynamicDetailView(DynamicView, RetrieveAPIView):
    """
    List view with dynamic properties
    """

class DynamicListCRUDView(DynamicView, ListCreateAPIView):
    """
    List view with dynamic properties
    """

class DynamicDetailCRUDView(DynamicView, RetrieveUpdateDestroyAPIView):
    """
    List view with dynamic properties
    """

class SaveAllSerializer(mixins.RetrieveModelMixin,
                        mixins.UpdateModelMixin,
                        mixins.DestroyModelMixin,
                        mixins.CreateModelMixin,
                        GenericAPIView):

    def get(self, request, *args, **kwargs):
        return self.retrieve(request, *args, **kwargs)

    def post(self, request, *args, **kwargs):
        return self.create(request, *args, **kwargs)

    def put(self, request, *args, **kwargs):
        return self.update(request, *args, **kwargs)

    def patch(self, request, *args, **kwargs):
        return self.partial_update(request, *args, **kwargs)

    def delete(self, request, *args, **kwargs):
        return self.destroy(request, *args, **kwargs)
<|endoftext|>


<commit_before>'''
Created on 20/01/2011

v0.1 (C) Gerald Storer
MIT License

Based on JSON.minify.js:
https://github.com/getify/JSON.minify
'''

import re

def json_minify(json,strip_space=True):
    tokenizer=re.compile('"|(/\*)|(\*/)|(//)|\n|\r')
    in_string = False
    in_multiline_comment = False
    in_singleline_comment = False

    new_str = []
    from_index = 0 # from is a keyword in Python

    for match in re.finditer(tokenizer,json):

        if not in_multiline_comment and not in_singleline_comment:
            tmp2 = json[from_index:match.start()]
            if not in_string and strip_space:
                tmp2 = re.sub('[ \t\n\r]*','',tmp2) # replace only white space defined in standard
            new_str.append(tmp2)

        from_index = match.end()

        if match.group() == '"' and not in_multiline_comment and not in_singleline_comment:
            escaped = re.search('(\\\\)*$',json[:match.start()])
            if not in_string or escaped is None or len(escaped.group()) % 2 == 0:
                # start of string with ", or unescaped " character found to end string
                in_string = not in_string
            from_index -= 1 # include " character in next catch

        elif match.group() == '/*' and not in_string and not in_multiline_comment and not in_singleline_comment:
            in_multiline_comment = True
        elif match.group() == '*/' and not in_string and in_multiline_comment and not in_singleline_comment:
            in_multiline_comment = False
        elif match.group() == '//' and not in_string and not in_multiline_comment and not in_singleline_comment:
            in_singleline_comment = True
        elif (match.group() == '\n' or match.group() == '\r') and not in_string and not in_multiline_comment and in_singleline_comment:
            in_singleline_comment = False
        elif not in_multiline_comment and not in_singleline_comment and (
             match.group() not in ['\n','\r',' ','\t'] or not strip_space):
                new_str.append(match.group())

    new_str.append(json[from_index:])
    return ''.join(new_str)

if __name__ == '__main__':
    import json # requires Python 2.6+ to run tests

    def test_json(s):
        return json.loads(json_minify(s))

    test1 = '''// this is a JSON file with comments
{
    "foo": "bar",    // this is cool
    "bar": [
        "baz", "bum", "zam"
    ],
/* the rest of this document is just fluff
   in case you are interested. */
    "something": 10,
    "else": 20
}

/* NOTE: You can easily strip the whitespace and comments
   from such a file with the JSON.minify() project hosted
   here on github at http://github.com/getify/JSON.minify
*/
'''

    test1_res = '''{"foo":"bar","bar":["baz","bum","zam"],"something":10,"else":20}'''

    test2 = '''
{"/*":"*/","//":"",/*"//"*/"/*/"://
"//"}

'''
    test2_res = '''{"/*":"*/","//":"","/*/":"//"}'''

    test3 = r'''/*
this is a
multi line comment */{

"foo"
:
    "bar/*"// something
    ,    "b\"az":/*
something else */"blah"

}
'''
    test3_res = r'''{"foo":"bar/*","b\"az":"blah"}'''

    test4 = r'''{"foo": "ba\"r//", "bar\\": "b\\\"a/*z",
    "baz\\\\": /* yay */ "fo\\\\\"*/o"
}
'''
    test4_res = r'''{"foo":"ba\"r//","bar\\":"b\\\"a/*z","baz\\\\":"fo\\\\\"*/o"}'''

    assert test_json(test1) == json.loads(test1_res),'Failed test 1'
    assert test_json(test2) == json.loads(test2_res),'Failed test 2'
    assert test_json(test3) == json.loads(test3_res),'Failed test 3'
    assert test_json(test4) == json.loads(test4_res),'Failed test 4'
    if __debug__: # Don't print passed message if the asserts didn't run
        print 'Passed all tests'<commit_msg>deleted:    src/python/jterator/minify_json.py  - This JSON dialect has turned obsolete after the switch to YAML. Removing it.<commit_after><|endoftext|>


<commit_before>import numpy
import multiprocessing
import itertools
import scipy.sparse.linalg
from exp.util.GeneralLinearOperator import GeneralLinearOperator

XrData = 0
XrIndices = 0
XrIntptr = 0
XShape = 0
WArray = 0
WShape = 0

def initProcess(data, indices, indptr, shape, Warr, Wshp):
    global XrData
    XrData = data
    global XrIndices
    XrIndices = indices
    global XrIntptr
    XrIntptr = indptr
    global Xshape
    Xshape = shape

    global WArray
    WArray = Warr
    global WShape
    WShape = Wshp

def dot(args):
    X, w = args
    return X.dot(w)

def dot2(args):
    rowInds, i = args

    global XrData
    global XrIndices
    global XrIntptr
    global Xshape

    data = numpy.frombuffer(XrData, dtype=numpy.float)
    indices = numpy.frombuffer(XrIndices, dtype=numpy.int32)
    indptr = numpy.frombuffer(XrIntptr, dtype=numpy.int32)
    Xr = scipy.sparse.csr_matrix((data, indices, indptr), shape=Xshape)

    global WArray
    global WShape
    W = numpy.frombuffer(WArray, dtype=numpy.float).reshape(WShape)

    return Xr[rowInds[i]:rowInds[i+1], :].dot(W)

def dotT(args):
    X, w = args
    return X.T.dot(w)

def dotSVD(args):
    X, U, s, V, w = args
    return X.dot(w) + (U*s).dot(V.T.dot(w))

def dotSVDT(args):
    X, U, s, V, w = args
    return X.T.dot(w) + (V*s).dot(U.T.dot(w))

class LinOperatorUtils(object):
    """
    Some implementations of linear operators for various structured matrices.
    """
    def __init__(self):
        pass

    @staticmethod
    def sparseLowRankOp(X, U, s, V):
        if not scipy.sparse.issparse(X):
            raise ValueError("X matrix should be sparse")
        if X.shape[0] != U.shape[0] or X.shape[1] != V.shape[0]:
            raise ValueError("X and U s V^T should have the same shape")

        def matvec(w):
            return X.dot(w) + (U*s).dot(V.T.dot(w))

        def rmatvec(w):
            return X.T.dot(w) + (V*s).dot(U.T.dot(w))

        def matmat(W):
            return X.dot(W) + (U*s).dot(V.T.dot(W))

        def rmatmat(W):
            return X.T.dot(W) + (V*s).dot(U.T.dot(W))

        return GeneralLinearOperator(X.shape, matvec, rmatvec, matmat, rmatmat, dtype=X.dtype)

    @staticmethod
    def parallelSparseLowRankOp(X, U, s, V):
        numProcesses = multiprocessing.cpu_count()
        colInds = numpy.array(numpy.linspace(0, X.shape[1], numProcesses+1), numpy.int)

        def matvec(w):
            pool = multiprocessing.Pool(processes=numProcesses)
            paramList = []
            for i in range(numProcesses):
                paramList.append((X[:, colInds[i]:colInds[i+1]], U, s, V[colInds[i]:colInds[i+1], :], w[colInds[i]:colInds[i+1]]))

            iterator = pool.imap(dotSVD, paramList, chunksize=1)

            #iterator = itertools.imap(dotSVD, paramList)
            p = numpy.zeros(X.shape[0])

            for i in range(numProcesses):
                p += iterator.next()

            pool.terminate()

            return p

        def rmatvec(w):
            pool = multiprocessing.Pool(processes=numProcesses)
            paramList = []
            for i in range(numProcesses):
                paramList.append((X[:, colInds[i]:colInds[i+1]], U, s, V[colInds[i]:colInds[i+1], :], w))

            iterator = pool.imap(dotSVDT, paramList, chunksize=1)

            #iterator = itertools.imap(dotSVDT, paramList)
            p = numpy.zeros(X.shape[1])

            for i in range(numProcesses):
                p[colInds[i]:colInds[i+1]] = iterator.next()

            pool.terminate()

            return p

        def matmat(W):
            pool = multiprocessing.Pool(processes=numProcesses)
            paramList = []
            for i in range(numProcesses):
                paramList.append((X[:, colInds[i]:colInds[i+1]], U, s, V[colInds[i]:colInds[i+1], :], W[colInds[i]:colInds[i+1], :]))

            iterator = pool.map(dotSVD, paramList, chunksize=1)

            #iterator = itertools.imap(dotSVD, paramList)
            P = numpy.zeros((X.shape[0], W.shape[1]))

            for i in range(numProcesses):
                P += iterator[i]

            pool.terminate()

            return P

        def rmatmat(W):
            pool = multiprocessing.Pool(processes=numProcesses)
            paramList = []
            for i in range(numProcesses):
                paramList.append((X[:, colInds[i]:colInds[i+1]], U, s, V[colInds[i]:colInds[i+1], :], W))

            iterator = pool.imap(dotSVDT, paramList, chunksize=1)

            #iterator = itertools.imap(dotSVD, paramList)
            P = numpy.zeros((X.shape[1], W.shape[1]))

            for i in range(numProcesses):
                P[colInds[i]:colInds[i+1], :] = iterator.next()

            pool.terminate()

            return P

        return GeneralLinearOperator(X.shape, matvec, rmatvec, matmat, rmatmat, dtype=X.dtype)

    @staticmethod
    def parallelSparseOp(X):
        """
        Return the parallel linear operator corresponding to left and right multiply of
        csc_matrix X. Note that there is a significant overhead for creating and waiting
        for locked processes.
        """
        if not scipy.sparse.isspmatrix_csc(X):
            raise ValueError("Currently only supports csc_matrices")

        #This doubles memory here but saves memory when on many CPUs and results in faster calculations when we do matmat
        Xr = X.tocsr()
        numProcesses = multiprocessing.cpu_count()
        numJobs = numProcesses
        rowInds = numpy.array(numpy.linspace(0, X.shape[0], numJobs+1), numpy.int)
        colInds = numpy.array(numpy.linspace(0, X.shape[1], numJobs+1), numpy.int)

        XrData = multiprocessing.RawArray("d", numpy.array(Xr.data))
        XrIndices = multiprocessing.RawArray("i", Xr.indices)
        XrIndptr = multiprocessing.RawArray("i", Xr.indptr)

        def matvec(w):
            pool = multiprocessing.Pool(processes=numProcesses)
            paramList = []
            for i in range(numJobs):
                paramList.append((Xr[rowInds[i]:rowInds[i+1], :], w))

            iterator = pool.imap(dot, paramList, chunksize=1)
            #iterator = itertools.imap(dot, paramList)
            p = numpy.zeros(X.shape[0])

            for i in range(numJobs):
                p[rowInds[i]:rowInds[i+1]] = iterator.next()

            pool.terminate()
            return p

        def rmatvec(w):
            pool = multiprocessing.Pool(processes=numProcesses)
            paramList = []
            for i in range(numJobs):
                paramList.append((X[:, colInds[i]:colInds[i+1]], w))

            iterator = pool.imap(dotT, paramList, chunksize=1)
            #iterator = itertools.imap(dotT, paramList)
            p = numpy.zeros(X.shape[1])

            for i in range(numJobs):
                p[colInds[i]:colInds[i+1]] = iterator.next()

            pool.terminate()
            return p

        def matmat(W):
            WArray = multiprocessing.RawArray("d", W.flatten())
            pool = multiprocessing.Pool(processes=numProcesses, initializer=initProcess, initargs=(XrData, XrIndices, XrIndptr, X.shape, WArray, W.shape))
            params = []

            for i in range(numJobs):
                params.append((rowInds, i))

            iterator = pool.map(dot2, params)
            P = numpy.zeros((X.shape[0], W.shape[1]))

            for i in range(numJobs):
                P[rowInds[i]:rowInds[i+1], :] = iterator[i]

            return P

        def rmatmat(W):
            pool = multiprocessing.Pool(processes=numProcesses)
            paramList = []
            for i in range(numJobs):
                paramList.append((X[:, colInds[i]:colInds[i+1]], W))

            iterator = pool.imap(dotT, paramList, chunksize=1)
            #iterator = itertools.imap(dotT, paramList)
            P = numpy.zeros((X.shape[1], W.shape[1]))

            for i in range(numJobs):
                P[colInds[i]:colInds[i+1], :] = iterator.next()

            pool.terminate()
            return P

        return GeneralLinearOperator(X.shape, matvec, rmatvec, matmat, rmatmat, dtype=X.dtype)
    <commit_msg>Copy=False<commit_after>import numpy
import multiprocessing
import itertools
import scipy.sparse.linalg
from exp.util.GeneralLinearOperator import GeneralLinearOperator

XrData = 0
XrIndices = 0
XrIntptr = 0
XShape = 0
WArray = 0
WShape = 0

def initProcess(data, indices, indptr, shape, Warr, Wshp):
    global XrData
    XrData = data
    global XrIndices
    XrIndices = indices
    global XrIntptr
    XrIntptr = indptr
    global Xshape
    Xshape = shape

    global WArray
    WArray = Warr
    global WShape
    WShape = Wshp

def dot(args):
    X, w = args
    return X.dot(w)

def dot2(args):
    rowInds, i = args

    global XrData
    global XrIndices
    global XrIntptr
    global Xshape

    data = numpy.frombuffer(XrData, dtype=numpy.float)
    indices = numpy.frombuffer(XrIndices, dtype=numpy.int32)
    indptr = numpy.frombuffer(XrIntptr, dtype=numpy.int32)
    Xr = scipy.sparse.csr_matrix((data, indices, indptr), shape=Xshape, copy=False)

    global WArray
    global WShape
    W = numpy.frombuffer(WArray, dtype=numpy.float).reshape(WShape)

    return Xr[rowInds[i]:rowInds[i+1], :].dot(W)

def dotT(args):
    X, w = args
    return X.T.dot(w)

def dotSVD(args):
    X, U, s, V, w = args
    return X.dot(w) + (U*s).dot(V.T.dot(w))

def dotSVDT(args):
    X, U, s, V, w = args
    return X.T.dot(w) + (V*s).dot(U.T.dot(w))

class LinOperatorUtils(object):
    """
    Some implementations of linear operators for various structured matrices.
    """
    def __init__(self):
        pass

    @staticmethod
    def sparseLowRankOp(X, U, s, V):
        if not scipy.sparse.issparse(X):
            raise ValueError("X matrix should be sparse")
        if X.shape[0] != U.shape[0] or X.shape[1] != V.shape[0]:
            raise ValueError("X and U s V^T should have the same shape")

        def matvec(w):
            return X.dot(w) + (U*s).dot(V.T.dot(w))

        def rmatvec(w):
            return X.T.dot(w) + (V*s).dot(U.T.dot(w))

        def matmat(W):
            return X.dot(W) + (U*s).dot(V.T.dot(W))

        def rmatmat(W):
            return X.T.dot(W) + (V*s).dot(U.T.dot(W))

        return GeneralLinearOperator(X.shape, matvec, rmatvec, matmat, rmatmat, dtype=X.dtype)

    @staticmethod
    def parallelSparseLowRankOp(X, U, s, V):
        numProcesses = multiprocessing.cpu_count()
        colInds = numpy.array(numpy.linspace(0, X.shape[1], numProcesses+1), numpy.int)

        def matvec(w):
            pool = multiprocessing.Pool(processes=numProcesses)
            paramList = []
            for i in range(numProcesses):
                paramList.append((X[:, colInds[i]:colInds[i+1]], U, s, V[colInds[i]:colInds[i+1], :], w[colInds[i]:colInds[i+1]]))

            iterator = pool.imap(dotSVD, paramList, chunksize=1)

            #iterator = itertools.imap(dotSVD, paramList)
            p = numpy.zeros(X.shape[0])

            for i in range(numProcesses):
                p += iterator.next()

            pool.terminate()

            return p

        def rmatvec(w):
            pool = multiprocessing.Pool(processes=numProcesses)
            paramList = []
            for i in range(numProcesses):
                paramList.append((X[:, colInds[i]:colInds[i+1]], U, s, V[colInds[i]:colInds[i+1], :], w))

            iterator = pool.imap(dotSVDT, paramList, chunksize=1)

            #iterator = itertools.imap(dotSVDT, paramList)
            p = numpy.zeros(X.shape[1])

            for i in range(numProcesses):
                p[colInds[i]:colInds[i+1]] = iterator.next()

            pool.terminate()

            return p

        def matmat(W):
            pool = multiprocessing.Pool(processes=numProcesses)
            paramList = []
            for i in range(numProcesses):
                paramList.append((X[:, colInds[i]:colInds[i+1]], U, s, V[colInds[i]:colInds[i+1], :], W[colInds[i]:colInds[i+1], :]))

            iterator = pool.map(dotSVD, paramList, chunksize=1)

            #iterator = itertools.imap(dotSVD, paramList)
            P = numpy.zeros((X.shape[0], W.shape[1]))

            for i in range(numProcesses):
                P += iterator[i]

            pool.terminate()

            return P

        def rmatmat(W):
            pool = multiprocessing.Pool(processes=numProcesses)
            paramList = []
            for i in range(numProcesses):
                paramList.append((X[:, colInds[i]:colInds[i+1]], U, s, V[colInds[i]:colInds[i+1], :], W))

            iterator = pool.imap(dotSVDT, paramList, chunksize=1)

            #iterator = itertools.imap(dotSVD, paramList)
            P = numpy.zeros((X.shape[1], W.shape[1]))

            for i in range(numProcesses):
                P[colInds[i]:colInds[i+1], :] = iterator.next()

            pool.terminate()

            return P

        return GeneralLinearOperator(X.shape, matvec, rmatvec, matmat, rmatmat, dtype=X.dtype)

    @staticmethod
    def parallelSparseOp(X):
        """
        Return the parallel linear operator corresponding to left and right multiply of
        csc_matrix X. Note that there is a significant overhead for creating and waiting
        for locked processes.
        """
        if not scipy.sparse.isspmatrix_csc(X):
            raise ValueError("Currently only supports csc_matrices")

        #This doubles memory here but saves memory when on many CPUs and results in faster calculations when we do matmat
        Xr = X.tocsr()
        numProcesses = multiprocessing.cpu_count()
        numJobs = numProcesses
        rowInds = numpy.array(numpy.linspace(0, X.shape[0], numJobs+1), numpy.int)
        colInds = numpy.array(numpy.linspace(0, X.shape[1], numJobs+1), numpy.int)

        XrData = multiprocessing.RawArray("d", numpy.array(Xr.data))
        XrIndices = multiprocessing.RawArray("i", Xr.indices)
        XrIndptr = multiprocessing.RawArray("i", Xr.indptr)

        def matvec(w):
            pool = multiprocessing.Pool(processes=numProcesses)
            paramList = []
            for i in range(numJobs):
                paramList.append((Xr[rowInds[i]:rowInds[i+1], :], w))

            iterator = pool.imap(dot, paramList, chunksize=1)
            #iterator = itertools.imap(dot, paramList)
            p = numpy.zeros(X.shape[0])

            for i in range(numJobs):
                p[rowInds[i]:rowInds[i+1]] = iterator.next()

            pool.terminate()
            return p

        def rmatvec(w):
            pool = multiprocessing.Pool(processes=numProcesses)
            paramList = []
            for i in range(numJobs):
                paramList.append((X[:, colInds[i]:colInds[i+1]], w))

            iterator = pool.imap(dotT, paramList, chunksize=1)
            #iterator = itertools.imap(dotT, paramList)
            p = numpy.zeros(X.shape[1])

            for i in range(numJobs):
                p[colInds[i]:colInds[i+1]] = iterator.next()

            pool.terminate()
            return p

        def matmat(W):
            WArray = multiprocessing.RawArray("d", W.flatten())
            pool = multiprocessing.Pool(processes=numProcesses, initializer=initProcess, initargs=(XrData, XrIndices, XrIndptr, X.shape, WArray, W.shape))
            params = []

            for i in range(numJobs):
                params.append((rowInds, i))

            iterator = pool.map(dot2, params)
            P = numpy.zeros((X.shape[0], W.shape[1]))

            for i in range(numJobs):
                P[rowInds[i]:rowInds[i+1], :] = iterator[i]

            return P

        def rmatmat(W):
            pool = multiprocessing.Pool(processes=numProcesses)
            paramList = []
            for i in range(numJobs):
                paramList.append((X[:, colInds[i]:colInds[i+1]], W))

            iterator = pool.imap(dotT, paramList, chunksize=1)
            #iterator = itertools.imap(dotT, paramList)
            P = numpy.zeros((X.shape[1], W.shape[1]))

            for i in range(numJobs):
                P[colInds[i]:colInds[i+1], :] = iterator.next()

            pool.terminate()
            return P

        return GeneralLinearOperator(X.shape, matvec, rmatvec, matmat, rmatmat, dtype=X.dtype)
    <|endoftext|>