<commit_before>
import requests
from config import config
from comum.default_headers import default_headers
from dasboards.widgets import Widgets
from utils import Realtime
from _share import share

class Dashboard:

	def __init__(self, acc_token):
		self.token = acc_token
		self.default_options = {"json":True,"headers":default_headers(self)}

	# List Dashboards
	def list:
		url = '{api_url}/dashbaord'.format(api_url=config.api_url)
		method = "GET"

		options = this.default_options
		options["url"] = url
		options["method"] = method
		return requests(options)

	# Create a Dashboard
	def create(data):
		data = data if data else {}
		url = '{api_url}/dashbaord'.format(api_url=config.api_url)
		method = "POST"

		options = this.default_options
		options["url"] = url
		options["method"] = method
		options["data"] = data
		return requests(options)

	# Edit a Dashboard
	def edit(dashboard_id, data):
		data = data if data else {}
		url = '{api_url}/dashbaord/{dashboard_id}'.format(api_url=config.api_url, dashboard_id=dashboard_id)
		method = "PUT"

		options = this.default_options
		options["url"] = url
		options["method"] = method
		options["data"] = data
		return requests(options)

	# Delete a Dashboard
	def delete(dashboard_id):
		url = '{api_url}/dashbaord/{dashboard_id}'.format(api_url=config.api_url, dashboard_id=dashboard_id)
		method = "DELETE"

		options = this.default_options
		options["url"] = url
		options["method"] = method
		return requests(options)

	# Get Information for a Dashboard
	def info(dashboard_id):
		if(!dashbaord or dashboard_id == ''):
			return Promise
		url = '{api_url}/dashbaord/{dashboard_id}'.format(api_url=config.api_url, dashboard_id=dashboard_id)
		method = "GET"

		options = this.default_options
		options["url"] = url
		options["method"] = method
		return requests(options)

	# Get Information for a Dashboard
	def listeneing(dashboard_id, func, realtime):
		if(!dashbaord or dashboard_id == ''):
			return Promise

		if(!this.realtime and !realtime):
			self.realtime = Realtime(self.token)
		realtime =  realtime if realtime or self.realtime
		realtime.get_socket.on('{dashboard:{dashboard_id}'.format(dashboard_id=dashboard_id), func)

		return Promise

	# Stop listening to a dashboard by its ID
	def stopListening(id, realtime):
		if(!self.realtime and !realtime):
			return None

		realtime =  realtime if realtime or self.realtime
		realtime.get_socket.off("dashboard:{id}".format(id=id))

	# Get share list of the dashboard
	def shareList(dashboard_id):
		if(!dashboard_id or dashboard_id == ''):
			return Promise
		return share.list("dashboard",dashboard_id,self.default_options)

	# Share the dashboard with another person
	def shareSendInvite(dashboard_id, data):
		data = data if data else {}
		if(!dashboard_id or dashboard_id == ''):
			return Promise
		elif !data.email:
			return Promise
		return share.invite("dashboard",dashboard_id,this.default_options)

	# Change permssions of bucket
	def shareEdit(share_id, data):
		data = data if data else {}
		if(!share_id or share_id == ''):
			return Promise
		elif !data.email:
			return Promise
		return share.edit("dashboard",share_id,this.default_options)

	# Remove share of a bucket
	def shareDelete(share_id):
		if(!share_id or share_id == ''):
			return Promise
		return share.remove("dashboard",share_id,this.default_options)

	# Generate new public token for the dashboard
	def genPublicToken(dashboard_id):
		if(!dashboard_id or dashboard_id == ''):
			return Promise

		url = "{api_url}/dashboard/{dashboard_id}/share/public".format(api_url=config.api_url,dashboard_id=dashboard_id)
		method = "GET"

		options = this.default_options
		options["url"] = url
		options["method"] = method
		return requests(options)

	# Clone the dashboard with special parameters
	def shareClone(dashboard_id, data):
		if (!dashboard_id or dashboard_id == ''):
			return Promise
		elif !data.email:
			return Promise

		url = "{api_url}/dashboard/{dashboard_id}/share/copy".format(api_url=config.api_url,dashboard_id=dashboard_id)
		method = "POST"

		options = this.default_options
		options["url"] = url
		options["method"] = method
		return requests(options)

	def get_widgets():
		return Widgets(self.token)

<commit_msg>Updates to tago dashboards.py<commit_after>
import requests # Used to make HTTP requests
import json # Used to parse JSON
import os # Used to infer environment variables
import _share as share
from socket import TagoRealTime
from dashboard.widgets import Widgets

API_TAGO = os.environ.get('TAGO_SERVER') or 'https://api.tago.io'
REALTIME = os.environ.get('TAGO_REALTIME') or 'https://realtime.tago.io'

class Dashboard:

	def __init__(self, acc_token):
		self.token = acc_token
		self.default_headers = { 'content-type': 'application/json', 'Device-Token': token }
		self.default_options = {"json":True,"headers":self.default_headers}

	# List Dashboards
	def list:
		return requests.get('{api_endpoint}/dashboard'.format(api_endpoint=API_TAGO), headers=self.default_headers).json()

	# Create a Dashboard
	def create(data):
		data = data if data else {}
    	return requests.post('{api_endpoint}/dashboard'.format(api_endpoint=API_TAGO), headers=self.default_headers, data=json.dumps(data)).json()

	# Edit a Dashboard
	def edit(dashboard_id, data):
		data = data if data else {}
    	return requests.put('{api_endpoint}/dashboard/{dashboard_id}'.format(api_endpoint=API_TAGO, dashboard_id=dashboard_id), headers=self.default_headers, data=json.dumps(data)).json()

	# Delete a Dashboard
	def delete(dashboard_id):
		return requests.delete('{api_endpoint}/dashboard/{dashboard_id}'.format(api_endpoint=API_TAGO, dashboard_id=dashboard_id), headers=self.default_headers).json()

	# Get Information for a Dashboard
	def info(dashboard_id):
		if dashboard_id is None or dashboard_id == '':
		    return None # Dashboard ID parameter is obligatory.
	    return requests.get('{api_endpoint}/dashboard/{dashboard_id}'.format(api_endpoint=API_TAGO, dashboard_id=dashboard_id), headers=self.default_headers).json()

	# Get Information for a Dashboard
	def listening(dashboard_id, func, realtime):
		if dashboard_id is None or dashboard_id == '':
		    return None # Dashboard ID parameter is obligatory.
		if(!this.realtime and !realtime):
			self.realtime = TagoRealTime(TAGO_REALTIME, self.token, func)
		realtime = realtime if realtime else self.realtime
		realtime.get_socket().on('{dashboard:{dashboard_id}'.format(dashboard_id=dashboard_id), func)
		return "Listening to Dashboard "+dashboard_id

	# Stop listening to a dashboard by its ID
	def stopListening(id, realtime):
		if(!self.realtime and !realtime):
			return None
		realtime =  realtime if realtime else self.realtime
		realtime.get_socket().off("dashboard:{id}".format(id=id))

	# Get share list of the dashboard
	def shareList(dashboard_id):
		if dashboard_id is None or dashboard_id == '':
		    return None # Dashboard ID parameter is obligatory.
		return share.list("dashboard",dashboard_id,self.default_options)

	# Share the dashboard with another person
	def shareSendInvite(dashboard_id, data):
		data = data if data else {}
		if dashboard_id is None or dashboard_id == '':
		    return None # Dashboard ID parameter is obligatory.
		elif data['email'] is None or data['email'] == '':
		    return None # data email parameter is obligatory.
		return share.invite("dashboard",dashboard_id,this.default_options)

	# Change permssions of bucket
	def shareEdit(share_id, data):
		data = data if data else {}
		if share_id is None or share_id == '':
		    return None # Share ID parameter is obligatory.
		elif data['email'] is None or data['email'] == '':
		    return None # data email parameter is obligatory.
		return share.edit("dashboard",share_id,this.default_options)

	# Remove share of a bucket
	def shareDelete(share_id):
		if share_id is None or share_id == '':
		    return None # Share ID parameter is obligatory.
		return share.remove("dashboard",share_id,this.default_options)

	# Generate new public token for the dashboard
	def genPublicToken(dashboard_id):
		if dashboard_id is None or dashboard_id == '':
		    return self.list() # Dashboard ID parameter is obligatory.
		return requests.get('{api_endpoint}/dashboard/{dashboard_id}/share/public'.format(api_endpoint=API_TAGO, dashboard_id=dashboard_id), headers=self.default_headers).json()

	# Clone the dashboard with special parameters
	def shareClone(dashboard_id, data):
		if dashboard_id is None or dashboard_id == '':
		    return self.list() # Dashboard ID parameter is obligatory.
		elif data.email is None:
		    return self.list() # data.email parameter is obligatory.
		return requests.post('{api_endpoint}/dashboard/{dashboard_id}/share/copy'.format(api_endpoint=API_TAGO, dashboard_id=dashboard_id), headers=self.default_headers).json()

	def get_widgets():
		return Widgets(self.token)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from django.db.models.loading import get_model

from rest_framework.response import Response
from rest_framework import viewsets

from taiga.base import exceptions as excp
from taiga.projects.userstories.serializers import UserStorySerializer
from taiga.projects.tasks.serializers import TaskSerializer
from taiga.projects.issues.serializers import IssueSerializer
from taiga.projects.wiki.serializers import WikiPageSerializer

class SearchViewSet(viewsets.ViewSet):
    def list(self, request, **kwargs):
        project_model = get_model("projects", "Project")
        text = request.QUERY_PARAMS.get('text', "")
        project_id = request.QUERY_PARAMS.get('project', None)

        try:
            project = self._get_project(project_id)
        except (project_model.DoesNotExist, TypeError):
            raise excp.PermissionDenied({"detail": "Wrong project id"})

        result = {
            "userstories": self._search_user_stories(project, text),
            "tasks": self._search_tasks(project, text),
            "issues": self._search_issues(project, text),
            "wikipages": self._search_wiki_pages(project, text)
        }

        result["count"] = sum(map(lambda x: len(x), result.values()))
        return Response(result)

    def _get_project(self, project_id):
        project_model = get_model("projects", "Project")
        own_projects = (project_model.objects
                            .filter(members=self.request.user))

        return own_projects.get(pk=project_id)

    def _search_user_stories(self, project, text):
        where_clause = ("to_tsvector(userstories_userstory.subject || "
                        "userstories_userstory.description) @@ plainto_tsquery(%s)")

        model_cls = get_model("userstories", "UserStory")
        queryset = (model_cls.objects
                        .extra(where=[where_clause], params=[text])
                        .filter(project_id=project.pk)[:50])

        serializer = UserStorySerializer(queryset, many=True)
        return serializer.data

    def _search_tasks(self, project, text):
        where_clause = ("to_tsvector(tasks_task.subject || tasks_task.description) "
                        "@@ plainto_tsquery(%s)")

        model_cls = get_model("tasks", "Task")
        queryset = (model_cls.objects
                        .extra(where=[where_clause], params=[text])
                        .filter(project_id=project.pk)[:50])

        serializer = TaskSerializer(queryset, many=True)
        return serializer.data

    def _search_issues(self, project, text):
        where_clause = ("to_tsvector(issues_issue.subject || issues_issue.description) "
                        "@@ plainto_tsquery(%s)")

        model_cls = get_model("issues", "Issue")
        queryset = (model_cls.objects
                        .extra(where=[where_clause], params=[text])
                        .filter(project_id=project.pk)[:50])

        serializer = IssueSerializer(queryset, many=True)
        return serializer.data

    def _search_wiki_pages(self, project, text):
        where_clause = ("to_tsvector(wiki_wikipage.slug || wiki_wikipage.content) "
                        "@@ plainto_tsquery(%s)")

        model_cls = get_model("wiki", "WikiPage")
        queryset = (model_cls.objects
                        .extra(where=[where_clause], params=[text])
                        .filter(project_id=project.pk)[:50])

        serializer = WikiPageSerializer(queryset, many=True)
        return serializer.data

<commit_msg>Working the search getting all optionally<commit_after># -*- coding: utf-8 -*-

from django.db.models.loading import get_model

from rest_framework.response import Response
from rest_framework import viewsets

from taiga.base import exceptions as excp
from taiga.projects.userstories.serializers import UserStorySerializer
from taiga.projects.tasks.serializers import TaskSerializer
from taiga.projects.issues.serializers import IssueSerializer
from taiga.projects.wiki.serializers import WikiPageSerializer

class SearchViewSet(viewsets.ViewSet):
    def list(self, request, **kwargs):
        project_model = get_model("projects", "Project")
        text = request.QUERY_PARAMS.get('text', "")
        get_all = request.QUERY_PARAMS.get('get_all', False)
        project_id = request.QUERY_PARAMS.get('project', None)

        try:
            project = self._get_project(project_id)
        except (project_model.DoesNotExist, TypeError):
            raise excp.PermissionDenied({"detail": "Wrong project id"})

        result = {
            "userstories": self._search_user_stories(project, text, get_all),
            "tasks": self._search_tasks(project, text, get_all),
            "issues": self._search_issues(project, text, get_all),
            "wikipages": self._search_wiki_pages(project, text, get_all)
        }

        result["count"] = sum(map(lambda x: len(x), result.values()))
        return Response(result)

    def _get_project(self, project_id):
        project_model = get_model("projects", "Project")
        own_projects = (project_model.objects
                            .filter(members=self.request.user))

        return own_projects.get(pk=project_id)

    def _search_user_stories(self, project, text, get_all):
        where_clause = ("to_tsvector(userstories_userstory.subject || "
                        "userstories_userstory.description) @@ plainto_tsquery(%s)")

        model_cls = get_model("userstories", "UserStory")
        if get_all != "false" and text == '':
            queryset = model_cls.objects.filter(project_id=project.pk)
        else:
            queryset = (model_cls.objects
                            .extra(where=[where_clause], params=[text])
                            .filter(project_id=project.pk)[:50])

        serializer = UserStorySerializer(queryset, many=True)
        return serializer.data

    def _search_tasks(self, project, text, get_all):
        where_clause = ("to_tsvector(tasks_task.subject || tasks_task.description) "
                        "@@ plainto_tsquery(%s)")

        model_cls = get_model("tasks", "Task")
        if get_all != "false" and text == '':
            queryset = model_cls.objects.filter(project_id=project.pk)
        else:
            queryset = (model_cls.objects
                            .extra(where=[where_clause], params=[text])
                            .filter(project_id=project.pk)[:50])

        serializer = TaskSerializer(queryset, many=True)
        return serializer.data

    def _search_issues(self, project, text, get_all):
        where_clause = ("to_tsvector(issues_issue.subject || issues_issue.description) "
                        "@@ plainto_tsquery(%s)")

        model_cls = get_model("issues", "Issue")
        if get_all != "false" and text == '':
            queryset = model_cls.objects.filter(project_id=project.pk)
        else:
            queryset = (model_cls.objects
                            .extra(where=[where_clause], params=[text])
                            .filter(project_id=project.pk)[:50])

        serializer = IssueSerializer(queryset, many=True)
        return serializer.data

    def _search_wiki_pages(self, project, text, get_all):
        where_clause = ("to_tsvector(wiki_wikipage.slug || wiki_wikipage.content) "
                        "@@ plainto_tsquery(%s)")

        model_cls = get_model("wiki", "WikiPage")
        if get_all != "false" and text == '':
            queryset = model_cls.objects.filter(project_id=project.pk)
        else:
            queryset = (model_cls.objects
                            .extra(where=[where_clause], params=[text])
                            .filter(project_id=project.pk)[:50])

        serializer = WikiPageSerializer(queryset, many=True)
        return serializer.data

<|endoftext|>


<commit_before>from importlib import import_module

from django.conf import settings
from django.core import urlresolvers
from django.core.exceptions import ImproperlyConfigured

from models import Comment
from forms import CommentForm

COMMENT_APP = 'tango_comments'

def get_comment_app():
    """
    Get the comment app as defined in the settings
    """
    # Make sure the app's in INSTALLED_APPS
    if COMMENT_APP not in settings.INSTALLED_APPS:
        raise ImproperlyConfigured("%s must be in INSTALLED_APPS" % COMMENT_APP)

    # Try to import the package
    try:
        package = import_module(COMMENT_APP)
    except ImportError:
        raise ImproperlyConfigured("% is not installed." % COMMENT_APP)

    return package

def get_model():
    """
    Returns the comment model class.
    """
    if hasattr(get_comment_app(), "get_model"):
        return get_comment_app().get_model()
    else:
        return Comment

def get_form():
    """
    Returns the comment ModelForm class.
    """
    if hasattr(get_comment_app(), "get_form"):
        return get_comment_app().get_form()
    else:
        return CommentForm

def get_form_target():
    """
    Returns the target URL for the comment form submission view.
    """
    if hasattr(get_comment_app(), "get_form_target"):
        return get_comment_app().get_form_target()
    else:
        return urlresolvers.reverse("tango_comments.views.comments.post_comment")

def get_flag_url(comment):
    """
    Get the URL for the "flag this comment" view.
    """
    if hasattr(get_comment_app(), "get_flag_url"):
        return get_comment_app().get_flag_url(comment)
    else:
        return urlresolvers.reverse("tango_comments.views.moderation.flag",
                                    args=(comment.id,))

def get_delete_url(comment):
    """
    Get the URL for the "delete this comment" view.
    """
    return urlresolvers.reverse("tango_comments.views.moderation.delete", args=(comment.id,))

def get_approve_url(comment):
    """
    Get the URL for the "approve this comment from moderation" view.
    """
    return urlresolvers.reverse("tango_comments.views.moderation.approve", args=(comment.id,))
<commit_msg>ok, comments, I don't like you and you don't like me...<commit_after>from importlib import import_module

from django.conf import settings
from django.core import urlresolvers
from django.core.exceptions import ImproperlyConfigured

from forms import CommentForm

COMMENT_APP = 'tango_comments'

def get_comment_app():
    """
    Get the comment app as defined in the settings
    """
    # Make sure the app's in INSTALLED_APPS
    if COMMENT_APP not in settings.INSTALLED_APPS:
        raise ImproperlyConfigured("%s must be in INSTALLED_APPS" % COMMENT_APP)

    # Try to import the package
    try:
        package = import_module(COMMENT_APP)
    except ImportError:
        raise ImproperlyConfigured("% is not installed." % COMMENT_APP)

    return package

def get_model():
    """
    Returns the comment model class.
    """
    return get_comment_app().get_model()

def get_form():
    """
    Returns the comment ModelForm class.
    """
    if hasattr(get_comment_app(), "get_form"):
        return get_comment_app().get_form()
    else:
        return CommentForm

def get_form_target():
    """
    Returns the target URL for the comment form submission view.
    """
    if hasattr(get_comment_app(), "get_form_target"):
        return get_comment_app().get_form_target()
    else:
        return urlresolvers.reverse("tango_comments.views.comments.post_comment")

def get_flag_url(comment):
    """
    Get the URL for the "flag this comment" view.
    """
    if hasattr(get_comment_app(), "get_flag_url"):
        return get_comment_app().get_flag_url(comment)
    else:
        return urlresolvers.reverse("tango_comments.views.moderation.flag",
                                    args=(comment.id,))

def get_delete_url(comment):
    """
    Get the URL for the "delete this comment" view.
    """
    return urlresolvers.reverse("tango_comments.views.moderation.delete", args=(comment.id,))

def get_approve_url(comment):
    """
    Get the URL for the "approve this comment from moderation" view.
    """
    return urlresolvers.reverse("tango_comments.views.moderation.approve", args=(comment.id,))
<|endoftext|>


<commit_before># -*- coding:utf8 -*-
# File   : init.py
# Author : Jiayuan Mao
# Email  : maojiayuan@gmail.com
# Date   : 1/25/17
#
# This file is part of TensorArtist

import os
import sys
import resource

__all__ = ['release_syslim', 'tune_opencv', 'tune_tensorflow', 'initialize_main']

def release_syslim():
    sys.setrecursionlimit(1000000)
    slim = 65536 * 1024
    resource.setrlimit(resource.RLIMIT_STACK, (slim, slim))

def tune_tensorflow():
    os.environ['TF_ENABLE_WINOGRAD_NONFUSED'] = '1'  # issue#9339
    os.environ['TF_AUTOTUNE_THRESHOLD'] = '3'   # use more warm-up

def tune_opencv():
    os.environ['OPENCV_OPENCL_RUNTIME'] = ''

def initialize_main():
    release_syslim()
    tune_tensorflow()

<commit_msg>[dev-mjy] ignore resource limit value error on some os<commit_after># -*- coding:utf8 -*-
# File   : init.py
# Author : Jiayuan Mao
# Email  : maojiayuan@gmail.com
# Date   : 1/25/17
#
# This file is part of TensorArtist

import os
import sys
import resource

__all__ = ['release_syslim', 'tune_opencv', 'tune_tensorflow', 'initialize_main']

def release_syslim():
    sys.setrecursionlimit(1000000)
    try:
        slim = 65536 * 1024
        resource.setrlimit(resource.RLIMIT_STACK, (slim, slim))
    except ValueError:
        pass

def tune_tensorflow():
    os.environ['TF_ENABLE_WINOGRAD_NONFUSED'] = '1'  # issue#9339
    os.environ['TF_AUTOTUNE_THRESHOLD'] = '3'   # use more warm-up

def tune_opencv():
    os.environ['OPENCV_OPENCL_RUNTIME'] = ''

def initialize_main():
    release_syslim()
    tune_tensorflow()

<|endoftext|>


<commit_before># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Script to test TF-TRT INT8 conversion without calibration on Mnist model."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.contrib.tensorrt.python import trt_convert
# pylint: disable=unused-import
from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
# pylint: enable=unused-import
from tensorflow.core.protobuf import config_pb2
from tensorflow.python import data
from tensorflow.python import keras
from tensorflow.python.estimator.estimator import Estimator
from tensorflow.python.estimator.model_fn import EstimatorSpec
from tensorflow.python.estimator.model_fn import ModeKeys
from tensorflow.python.estimator.run_config import RunConfig
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import graph_util
from tensorflow.python.framework import importer
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.keras.datasets import mnist
from tensorflow.python.layers import layers
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gen_array_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import metrics
from tensorflow.python.ops import nn
from tensorflow.python.ops import variable_scope
from tensorflow.python.ops.losses import losses
from tensorflow.python.platform import test
from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.summary import summary
from tensorflow.python.training import saver
from tensorflow.python.training.adam import AdamOptimizer
from tensorflow.python.training.checkpoint_management import latest_checkpoint
from tensorflow.python.training.training_util import get_global_step

INPUT_NODE_NAME = 'input'
OUTPUT_NODE_NAME = 'output'

class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):

  def _BuildGraph(self, x):

    def _Quantize(x, r):
      x = gen_array_ops.quantize_and_dequantize_v2(x, -r, r)
      return x

    def _DenseLayer(x, num_inputs, num_outputs, quantization_range, name):
      """Dense layer with quantized outputs.

      Args:
        x: input to the dense layer
        num_inputs: number of input columns of x
        num_outputs: number of output columns
        quantization_range: the min/max range for quantization
        name: name of the variable scope

      Returns:
        The output of the layer.
      """
      with variable_scope.variable_scope(name):
        kernel = variable_scope.get_variable(
            'kernel',
            shape=[num_inputs, num_outputs],
            dtype=dtypes.float32,
            initializer=keras.initializers.glorot_uniform())
        bias = variable_scope.get_variable(
            'bias',
            shape=[num_outputs],
            dtype=dtypes.float32,
            initializer=keras.initializers.zeros())
        x = math_ops.matmul(x, kernel)
        x = _Quantize(x, quantization_range)
        x = nn.bias_add(x, bias)
        x = _Quantize(x, quantization_range)
      return x

    x = _Quantize(x, 1)
    # Conv + Bias + Relu6
    x = layers.conv2d(x, filters=32, kernel_size=3, use_bias=True)
    x = nn.relu6(x)
    # Conv + Bias + Relu6
    x = layers.conv2d(x, filters=64, kernel_size=3, use_bias=True)
    x = nn.relu6(x)
    # Reduce
    x = math_ops.reduce_mean(x, [1, 2])
    x = _Quantize(x, 6)
    # FC1
    x = _DenseLayer(x, 64, 512, 6, name='dense')
    x = nn.relu6(x)
    # FC2
    x = _DenseLayer(x, 512, 10, 25, name='dense_1')
    x = array_ops.identity(x, name=OUTPUT_NODE_NAME)
    return x

  def _GetGraphDef(self, use_trt, max_batch_size, model_dir):
    """Get the frozen mnist GraphDef.

    Args:
      use_trt: whether use TF-TRT to convert the graph.
      max_batch_size: the max batch size to apply during TF-TRT conversion.
      model_dir: the model directory to load the checkpoints.

    Returns:
      The frozen mnist GraphDef.
    """
    graph = ops.Graph()
    with self.session(graph=graph) as sess:
      with graph.device('/GPU:0'):
        x = array_ops.placeholder(
            shape=(None, 28, 28, 1), dtype=dtypes.float32, name=INPUT_NODE_NAME)
        self._BuildGraph(x)
      # Load weights
      mnist_saver = saver.Saver()
      checkpoint_file = latest_checkpoint(model_dir)
      mnist_saver.restore(sess, checkpoint_file)
      # Freeze
      graph_def = graph_util.convert_variables_to_constants(
          sess, sess.graph_def, output_node_names=[OUTPUT_NODE_NAME])
    # Convert with TF-TRT
    if use_trt:
      logging.info('Number of nodes before TF-TRT conversion: %d',
                   len(graph_def.node))
      graph_def = trt_convert.create_inference_graph(
          graph_def,
          outputs=[OUTPUT_NODE_NAME],
          max_batch_size=max_batch_size,
          precision_mode='INT8',
          max_workspace_size_bytes=4096 << 19,
          minimum_segment_size=2,
          use_calibration=False,
      )
      logging.info('Number of nodes after TF-TRT conversion: %d',
                   len(graph_def.node))
      num_engines = len(
          [1 for n in graph_def.node if str(n.op) == 'TRTEngineOp'])
      self.assertEqual(1, num_engines)
    return graph_def

  def _Run(self, is_training, use_trt, batch_size, num_epochs, model_dir):
    """Train or evaluate the model.

    Args:
      is_training: whether to train or evaluate the model. In training mode,
        quantization will be simulated where the quantize_and_dequantize_v2 are
        placed.
      use_trt: if true, use TRT INT8 mode for evaluation, which will perform
        real quantization. Otherwise use native TensorFlow which will perform
        simulated quantization. Ignored if is_training is True.
      batch_size: batch size.
      num_epochs: how many epochs to train. Ignored if is_training is False.
      model_dir: where to save or load checkpoint.

    Returns:
      The Estimator evaluation result.
    """
    # Get dataset
    train_data, test_data = mnist.load_data()

    def _PreprocessFn(x, y):
      x = math_ops.cast(x, dtypes.float32)
      x = array_ops.expand_dims(x, axis=2)
      x = 2.0 * (x / 255.0) - 1.0
      y = math_ops.cast(y, dtypes.int32)
      return x, y

    def _EvalInputFn():
      mnist_x, mnist_y = test_data
      dataset = data.Dataset.from_tensor_slices((mnist_x, mnist_y))
      dataset = dataset.apply(
          data.experimental.map_and_batch(
              map_func=_PreprocessFn,
              batch_size=batch_size,
              num_parallel_calls=8))
      dataset = dataset.repeat(count=1)
      iterator = dataset.make_one_shot_iterator()
      features, labels = iterator.get_next()
      return features, labels

    def _TrainInputFn():
      mnist_x, mnist_y = train_data
      dataset = data.Dataset.from_tensor_slices((mnist_x, mnist_y))
      dataset = dataset.shuffle(2 * len(mnist_x))
      dataset = dataset.apply(
          data.experimental.map_and_batch(
              map_func=_PreprocessFn,
              batch_size=batch_size,
              num_parallel_calls=8))
      dataset = dataset.repeat(count=num_epochs)
      iterator = dataset.make_one_shot_iterator()
      features, labels = iterator.get_next()
      return features, labels

    def _ModelFn(features, labels, mode):
      if is_training:
        logits_out = self._BuildGraph(features)
      else:
        graph_def = self._GetGraphDef(use_trt, batch_size, model_dir)
        logits_out = importer.import_graph_def(
            graph_def,
            input_map={INPUT_NODE_NAME: features},
            return_elements=[OUTPUT_NODE_NAME + ':0'],
            name='')[0]

      loss = losses.sparse_softmax_cross_entropy(
          labels=labels, logits=logits_out)
      summary.scalar('loss', loss)

      classes_out = math_ops.argmax(logits_out, axis=1, name='classes_out')
      accuracy = metrics.accuracy(
          labels=labels, predictions=classes_out, name='acc_op')
      summary.scalar('accuracy', accuracy[1])

      if mode == ModeKeys.EVAL:
        return EstimatorSpec(
            mode, loss=loss, eval_metric_ops={'accuracy': accuracy})
      elif mode == ModeKeys.TRAIN:
        optimizer = AdamOptimizer(learning_rate=1e-2)
        train_op = optimizer.minimize(loss, global_step=get_global_step())
        return EstimatorSpec(mode, loss=loss, train_op=train_op)

    config_proto = config_pb2.ConfigProto()
    config_proto.gpu_options.allow_growth = True
    estimator = Estimator(
        model_fn=_ModelFn,
        model_dir=model_dir if is_training else None,
        config=RunConfig(session_config=config_proto))

    if is_training:
      estimator.train(_TrainInputFn)
    results = estimator.evaluate(_EvalInputFn)
    logging.info('accuracy: %s', str(results['accuracy']))
    return results

  # To generate the checkpoint, set a different model_dir and call self._Run()
  # by setting is_training=True and num_epochs=1000, e.g.:
  # model_dir = '/tmp/quantization_mnist'
  # self._Run(
  #     is_training=True,
  #     use_trt=False,
  #     batch_size=128,
  #     num_epochs=100,
  #     model_dir=model_dir)
  def testEval(self):
    if not trt_convert.is_tensorrt_enabled():
      return
    model_dir = test.test_src_dir_path('contrib/tensorrt/test/testdata')

    accuracy_tf_native = self._Run(
        is_training=False,
        use_trt=False,
        batch_size=128,
        num_epochs=None,
        model_dir=model_dir)['accuracy']
    logging.info('accuracy_tf_native: %f', accuracy_tf_native)
    self.assertAllClose(accuracy_tf_native, 0.9662)

    if trt_convert.get_linked_tensorrt_version()[0] < 5:
      return

    accuracy_tf_trt = self._Run(
        is_training=False,
        use_trt=True,
        batch_size=128,
        num_epochs=None,
        model_dir=model_dir)['accuracy']
    logging.info('accuracy_tf_trt: %f', accuracy_tf_trt)
    self.assertAllClose(accuracy_tf_trt, 0.9677)

if __name__ == '__main__':
  test.main()
<commit_msg>Reduce max_workspace_size_bytes so the GPU memory allocation doesn't exceed the limit.<commit_after># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Script to test TF-TRT INT8 conversion without calibration on Mnist model."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.contrib.tensorrt.python import trt_convert
# pylint: disable=unused-import
from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
# pylint: enable=unused-import
from tensorflow.core.protobuf import config_pb2
from tensorflow.python import data
from tensorflow.python import keras
from tensorflow.python.estimator.estimator import Estimator
from tensorflow.python.estimator.model_fn import EstimatorSpec
from tensorflow.python.estimator.model_fn import ModeKeys
from tensorflow.python.estimator.run_config import RunConfig
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import graph_util
from tensorflow.python.framework import importer
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.keras.datasets import mnist
from tensorflow.python.layers import layers
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gen_array_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import metrics
from tensorflow.python.ops import nn
from tensorflow.python.ops import variable_scope
from tensorflow.python.ops.losses import losses
from tensorflow.python.platform import test
from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.summary import summary
from tensorflow.python.training import saver
from tensorflow.python.training.adam import AdamOptimizer
from tensorflow.python.training.checkpoint_management import latest_checkpoint
from tensorflow.python.training.training_util import get_global_step

INPUT_NODE_NAME = 'input'
OUTPUT_NODE_NAME = 'output'

class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):

  def _BuildGraph(self, x):

    def _Quantize(x, r):
      x = gen_array_ops.quantize_and_dequantize_v2(x, -r, r)
      return x

    def _DenseLayer(x, num_inputs, num_outputs, quantization_range, name):
      """Dense layer with quantized outputs.

      Args:
        x: input to the dense layer
        num_inputs: number of input columns of x
        num_outputs: number of output columns
        quantization_range: the min/max range for quantization
        name: name of the variable scope

      Returns:
        The output of the layer.
      """
      with variable_scope.variable_scope(name):
        kernel = variable_scope.get_variable(
            'kernel',
            shape=[num_inputs, num_outputs],
            dtype=dtypes.float32,
            initializer=keras.initializers.glorot_uniform())
        bias = variable_scope.get_variable(
            'bias',
            shape=[num_outputs],
            dtype=dtypes.float32,
            initializer=keras.initializers.zeros())
        x = math_ops.matmul(x, kernel)
        x = _Quantize(x, quantization_range)
        x = nn.bias_add(x, bias)
        x = _Quantize(x, quantization_range)
      return x

    x = _Quantize(x, 1)
    # Conv + Bias + Relu6
    x = layers.conv2d(x, filters=32, kernel_size=3, use_bias=True)
    x = nn.relu6(x)
    # Conv + Bias + Relu6
    x = layers.conv2d(x, filters=64, kernel_size=3, use_bias=True)
    x = nn.relu6(x)
    # Reduce
    x = math_ops.reduce_mean(x, [1, 2])
    x = _Quantize(x, 6)
    # FC1
    x = _DenseLayer(x, 64, 512, 6, name='dense')
    x = nn.relu6(x)
    # FC2
    x = _DenseLayer(x, 512, 10, 25, name='dense_1')
    x = array_ops.identity(x, name=OUTPUT_NODE_NAME)
    return x

  def _GetGraphDef(self, use_trt, max_batch_size, model_dir):
    """Get the frozen mnist GraphDef.

    Args:
      use_trt: whether use TF-TRT to convert the graph.
      max_batch_size: the max batch size to apply during TF-TRT conversion.
      model_dir: the model directory to load the checkpoints.

    Returns:
      The frozen mnist GraphDef.
    """
    graph = ops.Graph()
    with self.session(graph=graph) as sess:
      with graph.device('/GPU:0'):
        x = array_ops.placeholder(
            shape=(None, 28, 28, 1), dtype=dtypes.float32, name=INPUT_NODE_NAME)
        self._BuildGraph(x)
      # Load weights
      mnist_saver = saver.Saver()
      checkpoint_file = latest_checkpoint(model_dir)
      mnist_saver.restore(sess, checkpoint_file)
      # Freeze
      graph_def = graph_util.convert_variables_to_constants(
          sess, sess.graph_def, output_node_names=[OUTPUT_NODE_NAME])
    # Convert with TF-TRT
    if use_trt:
      logging.info('Number of nodes before TF-TRT conversion: %d',
                   len(graph_def.node))
      graph_def = trt_convert.create_inference_graph(
          graph_def,
          outputs=[OUTPUT_NODE_NAME],
          max_batch_size=max_batch_size,
          precision_mode='INT8',
          # There is a 2GB GPU memory limit for each test, so we set
          # max_workspace_size_bytes to 256MB to leave enough room for TF
          # runtime to allocate GPU memory.
          max_workspace_size_bytes=1 << 28,
          minimum_segment_size=2,
          use_calibration=False,
      )
      logging.info('Number of nodes after TF-TRT conversion: %d',
                   len(graph_def.node))
      num_engines = len(
          [1 for n in graph_def.node if str(n.op) == 'TRTEngineOp'])
      self.assertEqual(1, num_engines)
    return graph_def

  def _Run(self, is_training, use_trt, batch_size, num_epochs, model_dir):
    """Train or evaluate the model.

    Args:
      is_training: whether to train or evaluate the model. In training mode,
        quantization will be simulated where the quantize_and_dequantize_v2 are
        placed.
      use_trt: if true, use TRT INT8 mode for evaluation, which will perform
        real quantization. Otherwise use native TensorFlow which will perform
        simulated quantization. Ignored if is_training is True.
      batch_size: batch size.
      num_epochs: how many epochs to train. Ignored if is_training is False.
      model_dir: where to save or load checkpoint.

    Returns:
      The Estimator evaluation result.
    """
    # Get dataset
    train_data, test_data = mnist.load_data()

    def _PreprocessFn(x, y):
      x = math_ops.cast(x, dtypes.float32)
      x = array_ops.expand_dims(x, axis=2)
      x = 2.0 * (x / 255.0) - 1.0
      y = math_ops.cast(y, dtypes.int32)
      return x, y

    def _EvalInputFn():
      mnist_x, mnist_y = test_data
      dataset = data.Dataset.from_tensor_slices((mnist_x, mnist_y))
      dataset = dataset.apply(
          data.experimental.map_and_batch(
              map_func=_PreprocessFn,
              batch_size=batch_size,
              num_parallel_calls=8))
      dataset = dataset.repeat(count=1)
      iterator = dataset.make_one_shot_iterator()
      features, labels = iterator.get_next()
      return features, labels

    def _TrainInputFn():
      mnist_x, mnist_y = train_data
      dataset = data.Dataset.from_tensor_slices((mnist_x, mnist_y))
      dataset = dataset.shuffle(2 * len(mnist_x))
      dataset = dataset.apply(
          data.experimental.map_and_batch(
              map_func=_PreprocessFn,
              batch_size=batch_size,
              num_parallel_calls=8))
      dataset = dataset.repeat(count=num_epochs)
      iterator = dataset.make_one_shot_iterator()
      features, labels = iterator.get_next()
      return features, labels

    def _ModelFn(features, labels, mode):
      if is_training:
        logits_out = self._BuildGraph(features)
      else:
        graph_def = self._GetGraphDef(use_trt, batch_size, model_dir)
        logits_out = importer.import_graph_def(
            graph_def,
            input_map={INPUT_NODE_NAME: features},
            return_elements=[OUTPUT_NODE_NAME + ':0'],
            name='')[0]

      loss = losses.sparse_softmax_cross_entropy(
          labels=labels, logits=logits_out)
      summary.scalar('loss', loss)

      classes_out = math_ops.argmax(logits_out, axis=1, name='classes_out')
      accuracy = metrics.accuracy(
          labels=labels, predictions=classes_out, name='acc_op')
      summary.scalar('accuracy', accuracy[1])

      if mode == ModeKeys.EVAL:
        return EstimatorSpec(
            mode, loss=loss, eval_metric_ops={'accuracy': accuracy})
      elif mode == ModeKeys.TRAIN:
        optimizer = AdamOptimizer(learning_rate=1e-2)
        train_op = optimizer.minimize(loss, global_step=get_global_step())
        return EstimatorSpec(mode, loss=loss, train_op=train_op)

    config_proto = config_pb2.ConfigProto()
    config_proto.gpu_options.allow_growth = True
    estimator = Estimator(
        model_fn=_ModelFn,
        model_dir=model_dir if is_training else None,
        config=RunConfig(session_config=config_proto))

    if is_training:
      estimator.train(_TrainInputFn)
    results = estimator.evaluate(_EvalInputFn)
    logging.info('accuracy: %s', str(results['accuracy']))
    return results

  # To generate the checkpoint, set a different model_dir and call self._Run()
  # by setting is_training=True and num_epochs=1000, e.g.:
  # model_dir = '/tmp/quantization_mnist'
  # self._Run(
  #     is_training=True,
  #     use_trt=False,
  #     batch_size=128,
  #     num_epochs=100,
  #     model_dir=model_dir)
  def testEval(self):
    if not trt_convert.is_tensorrt_enabled():
      return
    model_dir = test.test_src_dir_path('contrib/tensorrt/test/testdata')

    accuracy_tf_native = self._Run(
        is_training=False,
        use_trt=False,
        batch_size=128,
        num_epochs=None,
        model_dir=model_dir)['accuracy']
    logging.info('accuracy_tf_native: %f', accuracy_tf_native)
    self.assertAllClose(accuracy_tf_native, 0.9662)

    if trt_convert.get_linked_tensorrt_version()[0] < 5:
      return

    accuracy_tf_trt = self._Run(
        is_training=False,
        use_trt=True,
        batch_size=128,
        num_epochs=None,
        model_dir=model_dir)['accuracy']
    logging.info('accuracy_tf_trt: %f', accuracy_tf_trt)
    self.assertAllClose(accuracy_tf_trt, 0.9677)

if __name__ == '__main__':
  test.main()
<|endoftext|>


<commit_before>""" Functions for calibration, including creation of gaintables, application of gaintables, and
merging gaintables.

"""

import copy
import logging

import numpy.linalg

from data_models.memory_data_models import GainTable, BlockVisibility, QA, assert_vis_gt_compatible
from data_models.memory_data_models import ReceptorFrame
from ..visibility.iterators import vis_timeslice_iter

log = logging.getLogger(__name__)

def gaintable_summary(gt: GainTable):
    """Return string summarizing the Gaintable

    """
    return "%s rows, %.3f GB" % (gt.data.shape, gt.size())

def create_gaintable_from_blockvisibility(vis: BlockVisibility, timeslice=None,
                                          frequencyslice: float = None, **kwargs) -> GainTable:
    """ Create gain table from visibility.

    This makes an empty gain table consistent with the BlockVisibility.

    :param vis: BlockVisibilty
    :param timeslice: Time interval between solutions (s)
    :param frequency_width: Frequency solution width (Hz)
    :return: GainTable

    """
    assert isinstance(vis, BlockVisibility), "vis is not a BlockVisibility: %r" % vis

    nants = vis.nants

    if timeslice is None or timeslice == 'auto':
        utimes = numpy.unique(vis.time)
        ntimes = len(utimes)
        gain_interval = numpy.zeros([ntimes])
        if ntimes > 1:
            gain_interval[:-1] = utimes[1:] - utimes[0:-1]
            gain_interval[-1] = utimes[-1] - utimes[-2]
            gain_interval[0] = gain_interval[1]
        else:
            gain_interval[...] = 1.0

    else:
        ntimes = numpy.ceil((numpy.max(vis.time) - numpy.min(vis.time)) / timeslice).astype('int')
        utimes = numpy.linspace(numpy.min(vis.time), numpy.max(vis.time), ntimes)
        gain_interval = timeslice * numpy.ones([ntimes])

    #    log.debug('create_gaintable_from_blockvisibility: times are %s' % str(utimes))
    #    log.debug('create_gaintable_from_blockvisibility: intervals are %s' % str(gain_interval))

    ntimes = len(utimes)
    ufrequency = numpy.unique(vis.frequency)
    nfrequency = len(ufrequency)

    receptor_frame = ReceptorFrame(vis.polarisation_frame.type)
    nrec = receptor_frame.nrec

    gainshape = [ntimes, nants, nfrequency, nrec, nrec]
    gain = numpy.ones(gainshape, dtype='complex')
    if nrec > 1:
        gain[..., 0, 1] = 0.0
        gain[..., 1, 0] = 0.0

    gain_weight = numpy.ones(gainshape)
    gain_time = utimes
    gain_frequency = ufrequency
    gain_residual = numpy.zeros([ntimes, nfrequency, nrec, nrec])

    gt = GainTable(gain=gain, time=gain_time, interval=gain_interval, weight=gain_weight, residual=gain_residual,
                   frequency=gain_frequency,
                   receptor_frame=receptor_frame, phasecentre=vis.phasecentre)

    assert isinstance(gt, GainTable), "gt is not a GainTable: %r" % gt
    assert_vis_gt_compatible(vis, gt)

    return gt

def apply_gaintable(vis: BlockVisibility, gt: GainTable, inverse=False, vis_slices=None, **kwargs) -> BlockVisibility:
    """Apply a gain table to a block visibility

    The corrected visibility is::

        V_corrected = {g_i * g_j^*}^-1 V_obs

    If the visibility data are polarised e.g. polarisation_frame("linear") then the inverse operator
    represents an actual inverse of the gains.

    :param vis: Visibility to have gains applied
    :param gt: Gaintable to be applied
    :param inverse: Apply the inverse (default=False)
    :return: input vis with gains applied

    """
    assert isinstance(vis, BlockVisibility), "vis is not a BlockVisibility: %r" % vis
    assert isinstance(gt, GainTable), "gt is not a GainTable: %r" % gt

    assert_vis_gt_compatible(vis, gt)

    if inverse:
        log.debug('apply_gaintable: Apply inverse gaintable')
    else:
        log.debug('apply_gaintable: Apply gaintable')

    is_scalar = gt.gain.shape[-2:] == (1, 1)
    if is_scalar:
        log.debug('apply_gaintable: scalar gains')

    for chunk, rows in enumerate(vis_timeslice_iter(vis, vis_slices=vis_slices)):
        if numpy.sum(rows) > 0:
            vistime = numpy.average(vis.time[rows])
            gaintable_rows = abs(gt.time - vistime) < gt.interval / 2.0

            # Lookup the gain for this set of visibilities
            gain = gt.data['gain'][gaintable_rows]
            gainwt = gt.data['weight'][gaintable_rows]

            # The shape of the mueller matrix is
            ntimes, nant, nchan, nrec, _ = gain.shape

            original = vis.vis[rows]
            originalwt = vis.weight[rows]
            applied = copy.deepcopy(original)
            appliedwt = copy.deepcopy(originalwt)
            for time in range(ntimes):
                antantwt = numpy.outer(gainwt, gainwt)
                if is_scalar:
                    if inverse:
                        lgain = numpy.ones_like(gain)
                        lgain[numpy.abs(gain) > 0.0] = 1.0 / gain[numpy.abs(gain) > 0.0]
                    else:
                        lgain = gain
                    clgain = numpy.conjugate(lgain)
                    for chan in range(nchan):
                        smueller = numpy.ma.outer(lgain[time, :, chan, 0],
                                                  clgain[time, :, chan, 0]).reshape([nant, nant])
                        applied[time, :, :, chan, 0] = original[time, :, :, chan, 0] * smueller
                        antantwt = numpy.outer(gainwt[time, :, chan, 0, 0], gainwt[time, :, chan, 0, 0])
                        applied[time, :, :, chan, 0][antantwt == 0.0] = 0.0
                        appliedwt[time, :, :, chan, 0][antantwt == 0.0] = 0.0
                else:
                    for a1 in range(vis.nants - 1):
                        for a2 in range(a1 + 1, vis.nants):
                            for chan in range(nchan):
                                mueller = numpy.kron(gain[time, a1, chan, :, :],
                                                     numpy.conjugate(gain[time, a2, chan, :, :]))
                                if inverse:
                                    # If the Mueller is singular, ignore it
                                    try:
                                        mueller = numpy.linalg.inv(mueller)
                                        applied[time, a2, a1, chan, :] = \
                                            numpy.matmul(mueller, original[time, a2, a1, chan, :])
                                    except numpy.linalg.linalg.LinAlgError:
                                        applied[time, a2, a1, chan, :] = \
                                            original[time, a2, a1, chan, :]
                                else:
                                    applied[time, a2, a1, chan, :] = \
                                        numpy.matmul(mueller, original[time, a2, a1, chan, :])
                                if (gainwt[time, a1, chan, 0, 0] <= 0.0) or (gainwt[time, a1, chan, 0, 0] <= 0.0):
                                    applied[time, a2, a1, chan, 0] = 0.0
                                    appliedwt[time, a2, a1, chan, 0] = 0.0

            vis.data['vis'][rows] = applied
    return vis

def append_gaintable(gt: GainTable, othergt: GainTable) -> GainTable:
    """Append othergt to gt

    :param gt:
    :param othergt:
    :return: GainTable gt + othergt
    """
    assert gt.receptor_frame == othergt.receptor_frame
    gt.data = numpy.hstack((gt.data, othergt.data))
    return gt

def copy_gaintable(gt: GainTable, zero=False):
    """Copy a GainTable

    Performs a deepcopy of the data array
    """

    if gt is None:
        return gt

    assert isinstance(gt, GainTable), gt

    newgt = copy.copy(gt)
    newgt.data = copy.deepcopy(gt.data)
    if zero:
        newgt.data['gt'][...] = 0.0
    return newgt

def create_gaintable_from_rows(gt: GainTable, rows: numpy.ndarray, makecopy=True) -> GainTable:
    """ Create a GainTable from selected rows

    :param gt: GainTable
    :param rows: Boolean array of row selection
    :param makecopy: Make a deep copy (True)
    :return: GainTable
    """

    if rows is None or numpy.sum(rows) == 0:
        return None

    assert len(rows) == gt.ntimes, "Length of rows does not agree with length of GainTable"

    assert isinstance(gt, GainTable), gt

    if makecopy:
        newgt = copy_gaintable(gt)
        newgt.data = copy.deepcopy(gt.data[rows])
        return newgt
    else:
        gt.data = copy.deepcopy(gt.data[rows])

        return gt

def qa_gaintable(gt: GainTable, context=None) -> QA:
    """Assess the quality of a gaintable

    :param gt:
    :return: AQ
    """
    agt = numpy.abs(gt.gain[gt.weight > 0.0])
    pgt = numpy.angle(gt.gain[gt.weight > 0.0])
    data = {'shape': gt.gain.shape,
            'maxabs-amp': numpy.max(agt),
            'minabs-amp': numpy.min(agt),
            'rms-amp': numpy.std(agt),
            'medianabs-amp': numpy.median(agt),
            'maxabs-phase': numpy.max(pgt),
            'minabs-phase': numpy.min(pgt),
            'rms-phase': numpy.std(pgt),
            'medianabs-phase': numpy.median(pgt),
            'residual': numpy.max(gt.residual)
            }
    return QA(origin='qa_gaintable', data=data, context=context)

def gaintable_plot(gt: GainTable, ax, title='', value='amp', **kwargs):
    """ Standard plot of gain table

    :param gt: Gaintable
    :param ax: matplotlib axes
    :param value: 'amp' or 'phase'
    :param kwargs:
    :return:
    """
    nants = gt.nants
    for ant in range(nants):
        amp = numpy.abs(gt.gain[:, ant, 0, 0, 0])
        if value == 'amp':
            ax.plot(gt.time[gt.time > 0.0][amp > 0.0], 1.0 / amp[amp > 0.0], '.')
        else:
            angle = numpy.angle(gt.gain[:, ant, 0, 0, 0])
            ax.plot(gt.time[gt[0].time > 0.0][amp > 0.0], angle[amp[amp > 0.0]], '.')

    ax.set_title(title)
    ax.set_xlabel('Time (s)')
    ax.set_ylabel(value)
<commit_msg>Improved gain time selection<commit_after>""" Functions for calibration, including creation of gaintables, application of gaintables, and
merging gaintables.

"""

import copy
import logging

import numpy.linalg

from data_models.memory_data_models import GainTable, BlockVisibility, QA, assert_vis_gt_compatible
from data_models.memory_data_models import ReceptorFrame
from ..visibility.iterators import vis_timeslice_iter

log = logging.getLogger(__name__)

def gaintable_summary(gt: GainTable):
    """Return string summarizing the Gaintable

    """
    return "%s rows, %.3f GB" % (gt.data.shape, gt.size())

def create_gaintable_from_blockvisibility(vis: BlockVisibility, timeslice=None,
                                          frequencyslice: float = None, **kwargs) -> GainTable:
    """ Create gain table from visibility.

    This makes an empty gain table consistent with the BlockVisibility.

    :param vis: BlockVisibilty
    :param timeslice: Time interval between solutions (s)
    :param frequency_width: Frequency solution width (Hz)
    :return: GainTable

    """
    assert isinstance(vis, BlockVisibility), "vis is not a BlockVisibility: %r" % vis

    nants = vis.nants

    if timeslice is None or timeslice == 'auto':
        timeslice = numpy.min(vis.integration_time)

    utimes = timeslice * numpy.unique(numpy.round(vis.time / timeslice))
    ntimes = len(utimes)
    gain_interval = timeslice * numpy.ones([ntimes])

    #    log.debug('create_gaintable_from_blockvisibility: times are %s' % str(utimes))
    #    log.debug('create_gaintable_from_blockvisibility: intervals are %s' % str(gain_interval))

    ntimes = len(utimes)
    ufrequency = numpy.unique(vis.frequency)
    nfrequency = len(ufrequency)

    receptor_frame = ReceptorFrame(vis.polarisation_frame.type)
    nrec = receptor_frame.nrec

    gainshape = [ntimes, nants, nfrequency, nrec, nrec]
    gain = numpy.ones(gainshape, dtype='complex')
    if nrec > 1:
        gain[..., 0, 1] = 0.0
        gain[..., 1, 0] = 0.0

    gain_weight = numpy.ones(gainshape)
    gain_time = utimes
    gain_frequency = ufrequency
    gain_residual = numpy.zeros([ntimes, nfrequency, nrec, nrec])

    gt = GainTable(gain=gain, time=gain_time, interval=gain_interval, weight=gain_weight, residual=gain_residual,
                   frequency=gain_frequency,
                   receptor_frame=receptor_frame, phasecentre=vis.phasecentre)

    assert isinstance(gt, GainTable), "gt is not a GainTable: %r" % gt
    assert_vis_gt_compatible(vis, gt)

    return gt

def apply_gaintable(vis: BlockVisibility, gt: GainTable, inverse=False, vis_slices=None, **kwargs) -> BlockVisibility:
    """Apply a gain table to a block visibility

    The corrected visibility is::

        V_corrected = {g_i * g_j^*}^-1 V_obs

    If the visibility data are polarised e.g. polarisation_frame("linear") then the inverse operator
    represents an actual inverse of the gains.

    :param vis: Visibility to have gains applied
    :param gt: Gaintable to be applied
    :param inverse: Apply the inverse (default=False)
    :return: input vis with gains applied

    """
    assert isinstance(vis, BlockVisibility), "vis is not a BlockVisibility: %r" % vis
    assert isinstance(gt, GainTable), "gt is not a GainTable: %r" % gt

    assert_vis_gt_compatible(vis, gt)

    if inverse:
        log.debug('apply_gaintable: Apply inverse gaintable')
    else:
        log.debug('apply_gaintable: Apply gaintable')

    is_scalar = gt.gain.shape[-2:] == (1, 1)
    if is_scalar:
        log.debug('apply_gaintable: scalar gains')

    for chunk, rows in enumerate(vis_timeslice_iter(vis, vis_slices=vis_slices)):
        if numpy.sum(rows) > 0:
            vistime = numpy.average(vis.time[rows])
            gaintable_rows = abs(gt.time - vistime) < gt.interval / 2.0

            # Lookup the gain for this set of visibilities
            gain = gt.data['gain'][gaintable_rows]
            gainwt = gt.data['weight'][gaintable_rows]

            # The shape of the mueller matrix is
            ntimes, nant, nchan, nrec, _ = gain.shape

            original = vis.vis[rows]
            originalwt = vis.weight[rows]
            applied = copy.deepcopy(original)
            appliedwt = copy.deepcopy(originalwt)
            for time in range(ntimes):
                antantwt = numpy.outer(gainwt, gainwt)
                if is_scalar:
                    if inverse:
                        lgain = numpy.ones_like(gain)
                        lgain[numpy.abs(gain) > 0.0] = 1.0 / gain[numpy.abs(gain) > 0.0]
                    else:
                        lgain = gain
                    clgain = numpy.conjugate(lgain)
                    for chan in range(nchan):
                        smueller = numpy.ma.outer(lgain[time, :, chan, 0],
                                                  clgain[time, :, chan, 0]).reshape([nant, nant])
                        applied[time, :, :, chan, 0] = original[time, :, :, chan, 0] * smueller
                        antantwt = numpy.outer(gainwt[time, :, chan, 0, 0], gainwt[time, :, chan, 0, 0])
                        applied[time, :, :, chan, 0][antantwt == 0.0] = 0.0
                        appliedwt[time, :, :, chan, 0][antantwt == 0.0] = 0.0
                else:
                    for a1 in range(vis.nants - 1):
                        for a2 in range(a1 + 1, vis.nants):
                            for chan in range(nchan):
                                mueller = numpy.kron(gain[time, a1, chan, :, :],
                                                     numpy.conjugate(gain[time, a2, chan, :, :]))
                                if inverse:
                                    # If the Mueller is singular, ignore it
                                    try:
                                        mueller = numpy.linalg.inv(mueller)
                                        applied[time, a2, a1, chan, :] = \
                                            numpy.matmul(mueller, original[time, a2, a1, chan, :])
                                    except numpy.linalg.linalg.LinAlgError:
                                        applied[time, a2, a1, chan, :] = \
                                            original[time, a2, a1, chan, :]
                                else:
                                    applied[time, a2, a1, chan, :] = \
                                        numpy.matmul(mueller, original[time, a2, a1, chan, :])
                                if (gainwt[time, a1, chan, 0, 0] <= 0.0) or (gainwt[time, a1, chan, 0, 0] <= 0.0):
                                    applied[time, a2, a1, chan, 0] = 0.0
                                    appliedwt[time, a2, a1, chan, 0] = 0.0

            vis.data['vis'][rows] = applied
    return vis

def append_gaintable(gt: GainTable, othergt: GainTable) -> GainTable:
    """Append othergt to gt

    :param gt:
    :param othergt:
    :return: GainTable gt + othergt
    """
    assert gt.receptor_frame == othergt.receptor_frame
    gt.data = numpy.hstack((gt.data, othergt.data))
    return gt

def copy_gaintable(gt: GainTable, zero=False):
    """Copy a GainTable

    Performs a deepcopy of the data array
    """

    if gt is None:
        return gt

    assert isinstance(gt, GainTable), gt

    newgt = copy.copy(gt)
    newgt.data = copy.deepcopy(gt.data)
    if zero:
        newgt.data['gt'][...] = 0.0
    return newgt

def create_gaintable_from_rows(gt: GainTable, rows: numpy.ndarray, makecopy=True) -> GainTable:
    """ Create a GainTable from selected rows

    :param gt: GainTable
    :param rows: Boolean array of row selection
    :param makecopy: Make a deep copy (True)
    :return: GainTable
    """

    if rows is None or numpy.sum(rows) == 0:
        return None

    assert len(rows) == gt.ntimes, "Length of rows does not agree with length of GainTable"

    assert isinstance(gt, GainTable), gt

    if makecopy:
        newgt = copy_gaintable(gt)
        newgt.data = copy.deepcopy(gt.data[rows])
        return newgt
    else:
        gt.data = copy.deepcopy(gt.data[rows])

        return gt

def qa_gaintable(gt: GainTable, context=None) -> QA:
    """Assess the quality of a gaintable

    :param gt:
    :return: AQ
    """
    agt = numpy.abs(gt.gain[gt.weight > 0.0])
    pgt = numpy.angle(gt.gain[gt.weight > 0.0])
    data = {'shape': gt.gain.shape,
            'maxabs-amp': numpy.max(agt),
            'minabs-amp': numpy.min(agt),
            'rms-amp': numpy.std(agt),
            'medianabs-amp': numpy.median(agt),
            'maxabs-phase': numpy.max(pgt),
            'minabs-phase': numpy.min(pgt),
            'rms-phase': numpy.std(pgt),
            'medianabs-phase': numpy.median(pgt),
            'residual': numpy.max(gt.residual)
            }
    return QA(origin='qa_gaintable', data=data, context=context)

def gaintable_plot(gt: GainTable, ax, title='', value='amp', **kwargs):
    """ Standard plot of gain table

    :param gt: Gaintable
    :param ax: matplotlib axes
    :param value: 'amp' or 'phase'
    :param kwargs:
    :return:
    """
    nants = gt.nants
    for ant in range(nants):
        amp = numpy.abs(gt.gain[:, ant, 0, 0, 0])
        if value == 'amp':
            ax.plot(gt.time[gt.time > 0.0][amp > 0.0], 1.0 / amp[amp > 0.0], '.')
        else:
            angle = numpy.angle(gt.gain[:, ant, 0, 0, 0])
            ax.plot(gt.time[gt.time > 0.0][amp > 0.0], angle[amp > 0.0], '.')

    ax.set_title(title)
    ax.set_xlabel('Time (s)')
    ax.set_ylabel(value)
<|endoftext|>


<commit_before># Create your views here.

from django.template import Context, loader
from django.http import HttpResponse
from django.core import serializers
from world.models import World, Fortune
from django.shortcuts import render
from ujson import dumps as uj_dumps
import random
import sys
from operator import attrgetter
from functools import partial

if sys.version_info[0] == 3:
  xrange = range

def json(request):
  response = {
    "message": "Hello, World!"
  }
  return HttpResponse(uj_dumps(response), mimetype="application/json")

def db(request):
  queries = int(request.GET.get('queries', 1))
  # worlds = []
  # it's not required to explicitly loop instead of using list comprehension
  #for i in range(queries):
    # get a random row, we know the ids are between 1 and 10000
    #worlds.append(World.objects.get(id=random.randint(1, 10000)))
  # instead we can do:
  #worlds = [World.objects.get(id=random.randint(1, 10000)) for i in range(queries)]

  # fun fact:  every dot-notation lookup calls some python magic under the hood.  Like every other code,
  # one can eliminate dereferences by storing the end dereferenced thing in an identifier
  g = World.objects.get
  #r = random.randint
  # but wait!  there's more!  if we're calling a function over and over with the same parameters,
  # we can use even more function magic.
  rp = partial(random.randint, 1, 10000)
  # now we're ready to write our awesome query iterator thingy
  # first of all, we know the id's correspond to the random number we're picking, so we can create
  # dictionaries on the fly instead of serializing later
  # by creating dicts, we don't need to user the model serializer, which is probably slow and only appropriate
  # for complicated serializations of joins and crazy query sets etc
  # test xrange vs range if the query number is gigantic
  worlds = uj_dumps([{'id' : r, 'randomNumber' : g(id=r).randomnumber} for r in [rp() for q in xrange(queries)]])
  return HttpResponse(worlds, mimetype="application/json")

def fortunes(request):
  fortunes = list(Fortune.objects.all())
  fortunes.append(Fortune(id=0, message="Additional message added at runtime."))

  fortunes = sorted(fortunes, key=attrgetter('message'))

  context = {'fortunes': fortunes}
  return render(request, 'fortunes.html', context)

def update(request):
  queries = int(request.GET.get('queries', 1))
  g = World.objects.get
  rp = partial(random.randint, 1, 10000)

  worlds = []
  for r in [rp() for q in xrange(queries)]:
    w = g(id=r)
    w.randomnumber=rp()
    w.save()

    worlds.append({'id' : r, 'randomNumber' : w.randomnumber})

  return HttpResponse(worlds, mimetype="application/json")
<commit_msg>django: Fix update test result format.<commit_after># Create your views here.

from django.template import Context, loader
from django.http import HttpResponse
from django.core import serializers
from world.models import World, Fortune
from django.shortcuts import render
from ujson import dumps as uj_dumps
import random
import sys
from operator import attrgetter
from functools import partial

if sys.version_info[0] == 3:
  xrange = range

def json(request):
  response = {
    "message": "Hello, World!"
  }
  return HttpResponse(uj_dumps(response), mimetype="application/json")

def db(request):
  queries = int(request.GET.get('queries', 1))
  # worlds = []
  # it's not required to explicitly loop instead of using list comprehension
  #for i in range(queries):
    # get a random row, we know the ids are between 1 and 10000
    #worlds.append(World.objects.get(id=random.randint(1, 10000)))
  # instead we can do:
  #worlds = [World.objects.get(id=random.randint(1, 10000)) for i in range(queries)]

  # fun fact:  every dot-notation lookup calls some python magic under the hood.  Like every other code,
  # one can eliminate dereferences by storing the end dereferenced thing in an identifier
  g = World.objects.get
  #r = random.randint
  # but wait!  there's more!  if we're calling a function over and over with the same parameters,
  # we can use even more function magic.
  rp = partial(random.randint, 1, 10000)
  # now we're ready to write our awesome query iterator thingy
  # first of all, we know the id's correspond to the random number we're picking, so we can create
  # dictionaries on the fly instead of serializing later
  # by creating dicts, we don't need to user the model serializer, which is probably slow and only appropriate
  # for complicated serializations of joins and crazy query sets etc
  # test xrange vs range if the query number is gigantic
  worlds = uj_dumps([{'id' : r, 'randomNumber' : g(id=r).randomnumber} for r in [rp() for q in xrange(queries)]])
  return HttpResponse(worlds, mimetype="application/json")

def fortunes(request):
  fortunes = list(Fortune.objects.all())
  fortunes.append(Fortune(id=0, message="Additional message added at runtime."))

  fortunes = sorted(fortunes, key=attrgetter('message'))

  context = {'fortunes': fortunes}
  return render(request, 'fortunes.html', context)

def update(request):
  queries = int(request.GET.get('queries', 1))
  g = World.objects.get
  rp = partial(random.randint, 1, 10000)

  worlds = []
  for r in [rp() for q in xrange(queries)]:
    w = g(id=r)
    w.randomnumber=rp()
    w.save()

    worlds.append({'id' : r, 'randomNumber' : w.randomnumber})

  return HttpResponse(uj_dumps(worlds), mimetype="application/json")
<|endoftext|>


<commit_before>import sys
sys.path.append('C:/Users/dmccloskey-sbrg/Documents/GitHub/SBaaS_base')
from SBaaS_base.postgresql_settings import postgresql_settings
from SBaaS_base.postgresql_orm import postgresql_orm

# read in the settings file
filename = 'C:/Users/dmccloskey-sbrg/Google Drive/SBaaS_settings/settings_metabolomics.ini';
pg_settings = postgresql_settings(filename);

# connect to the database from the settings file
pg_orm = postgresql_orm();
pg_orm.set_sessionFromSettings(pg_settings.database_settings);
session = pg_orm.get_session();
engine = pg_orm.get_engine();

# your app...
# SBaaS paths:
sys.path.append(pg_settings.datadir_settings['github']+'/SBaaS_base')
sys.path.append(pg_settings.datadir_settings['github']+'/SBaaS_LIMS')
sys.path.append(pg_settings.datadir_settings['github']+'/SBaaS_quantification')
sys.path.append(pg_settings.datadir_settings['github']+'/SBaaS_physiology')
sys.path.append(pg_settings.datadir_settings['github']+'/SBaaS_MFA')
sys.path.append(pg_settings.datadir_settings['github']+'/SBaaS_visualization')
sys.path.append(pg_settings.datadir_settings['github']+'/SBaaS_models')
sys.path.append(pg_settings.datadir_settings['github']+'/SBaaS_thermodynamics')
sys.path.append(pg_settings.datadir_settings['github']+'/SBaaS_COBRA')
# SBaaS dependencies paths:
sys.path.append(pg_settings.datadir_settings['github']+'/io_utilities')
sys.path.append(pg_settings.datadir_settings['github']+'/quantification_analysis')
sys.path.append(pg_settings.datadir_settings['github']+'/matplotlib_utilities')
sys.path.append(pg_settings.datadir_settings['github']+'/thermodynamics/thermodynamics')
sys.path.append(pg_settings.datadir_settings['github']+'/component-contribution')
sys.path.append(pg_settings.datadir_settings['github']+'/molmass')
sys.path.append(pg_settings.datadir_settings['github']+'/python_statistics')
sys.path.append(pg_settings.datadir_settings['github']+'/r_statistics')
sys.path.append(pg_settings.datadir_settings['github']+'/listDict')
sys.path.append(pg_settings.datadir_settings['github']+'/ddt_python')

#make the simulation table
from SBaaS_thermodynamics.stage03_quantification_simulation_execute import stage03_quantification_simulation_execute
exsimulation01 = stage03_quantification_simulation_execute(session,engine,pg_settings.datadir_settings);

##import the simulation parameters
#exsimulation01.import_dataStage03QuantificationSimulation_add(pg_settings.datadir_settings['workspace_data']+'/_input/141007_data_stage03_quantification_simulation.csv');
#exsimulation01.import_dataStage03QuantificationSimulationParameters_add(pg_settings.datadir_settings['workspace_data']+'/_input/141007_data_stage03_quantification_simulationParameters.csv')
#exsimulation01.import_dataStage03QuantificationSimulation_add(pg_settings.datadir_settings['workspace_data']+'/_input/151026_data_stage03_quantification_simulation01.csv');
#exsimulation01.import_dataStage03QuantificationSimulationParameters_add(pg_settings.datadir_settings['workspace_data']+'/_input/151026_data_stage03_quantification_simulationParameters01.csv')

analysis_ids = ['ALEWt01'];
simulation_ids = ['ALEWt01_iJO1366_ALEWt_irreversible_OxicEvo03Glc_0',
    'ALEWt01_iJO1366_ALEWt_irreversible_OxicEvo04Glc_0',
    'ALEWt01_iJO1366_ALEWt_irreversible_OxicEvo08Glc_0',
    'ALEWt01_iJO1366_ALEWt_irreversible_OxicEvo09Glc_0',
    'ALEWt01_iJO1366_ALEWt_irreversible_OxicWtGlc_0',
    ];
data_dir_I = 'C:/Users/dmccloskey-sbrg/Dropbox (UCSD SBRG)/MATLAB/tsampling';

#make the measuredData table
from SBaaS_thermodynamics.stage03_quantification_measuredData_execute import stage03_quantification_measuredData_execute
exmeasuredData01 = stage03_quantification_measuredData_execute(session,engine,pg_settings.datadir_settings)
exmeasuredData01.initialize_dataStage03_quantification_measuredData();

#make the COBRA table
from SBaaS_models.models_COBRA_execute import models_COBRA_execute
exCOBRA01 = models_COBRA_execute(session,engine,pg_settings.datadir_settings);
exCOBRA01.initialize_supportedTables();
exCOBRA01.initialize_COBRA_models();

#make the dG_r table
from SBaaS_thermodynamics.stage03_quantification_dG_r_execute import stage03_quantification_dG_r_execute
exdGr01 = stage03_quantification_dG_r_execute(session,engine,pg_settings.datadir_settings)
exdGr01.initialize_supportedTables();
exdGr01.initialize_tables()

#pre-load the models
thermomodels = exCOBRA01.get_models(model_ids_I=["iJO1366"]);

exdGr01.reset_dataStage03_quantification_dG_r_comparison(
    analysis_id_I='ALEsKOs01_0_evo04_0_11_evo04gnd'
    );

#perform a thermodynamic comparison
exdGr01.execute_compare_dG_r(
    analysis_id_I='ALEsKOs01_0_evo04_0_11_evo04gnd',
    simulation_id_base_I='ALEsKOs01_iJO1366_OxicEvo04EcoliGlc_0',
    simulation_ids_I=[],
    models_I=thermomodels,
    measured_concentration_coverage_criteria_I=0.5,
    measured_dG_f_coverage_criteria_I=0.99)

#make the COBRA table
from SBaaS_models.models_COBRA_execute import models_COBRA_execute
exCOBRA01 = models_COBRA_execute(session,engine,pg_settings.datadir_settings);
##reset 151026_iDM2015
#exCOBRA01.reset_COBRA_models('151026_iDM2015');
##make 151026_iDM2015
#exCOBRA01.make_modelFromRxnsAndMetsTables(
#        model_id_I='151026_iDM2015_netRxns',
#        #model_file_name_I=pg_settings.datadir_settings['workspace_data']+ '/models/150526_iDM2015.xml',
#        model_id_O='151026_iDM2015',date_O='2015-10-30 00:00:00',
#        ko_list=[],flux_dict={},
#        rxn_include_list=None,
#        description=None,
#        convert2irreversible_I=False,
#        revert2reversible_I=False,
#        convertPathway2individualRxns_I=True,
#        model_id_template_I='iJO1366_ALEWt',
#        pathway_model_id_I='iJO1366');

#test the model
#test_result = exCOBRA01.execute_testModel(model_id_I='151026_iDM2015');
#test_result = exCOBRA01.execute_testModel(model_id_I='151026_iDM2015_netRxns');
#pre-load the models
#thermomodels = exCOBRA01.get_models(model_ids_I=['151026_iDM2015']);
#test_O = exmeasuredData01.execute_testMeasuredFluxes(
#    experiment_id_I = 'ALEWt01',
#    models_I=thermomodels,
#    model_ids_I=['151026_iDM2015'],
#    diagnose_I=True);
#thermomodels = exCOBRA01.get_models(model_ids_I=['151026_iDM2015_netRxns']);
#test_O = exmeasuredData01.execute_testMeasuredFluxes(
#    experiment_id_I = 'ALEWt01',
#    models_I=thermomodels,
#    model_ids_I=['151026_iDM2015_netRxns'],
#    diagnose_I=True,
#    update_measuredFluxes_I=True,
#    adjustment_1_I=True,
#    adjustment_2_I=False,
#    );
#test_O = exmeasuredData01.execute_testMeasuredFluxes(
#    experiment_id_I = 'ALEWt01',
#    models_I=thermomodels,
#    model_ids_I=['151026_iDM2015_netRxns'],
#    diagnose_I=False,
#    update_measuredFluxes_I=False);

##151026_iDM2014
##make the tfba table
#from SBaaS_thermodynamics.stage03_quantification_tfba_execute import stage03_quantification_tfba_execute
#extfba01 = stage03_quantification_tfba_execute(session,engine,pg_settings.datadir_settings)
#extfba01.initialize_dataStage03_quantification_tfba();
#for simulation_id in simulation_ids:
#    ##reset previous simulations
#    ##extfba01.reset_dataStage03_quantification_tfba(simulation_id);
#    inconsistent_tcc=['IPDPS', 'IMPD','PPKr_reverse', 'NTD4', 'TPI', 'PGMT_reverse']
#    #load sampling points after sampling
#    extfba01.execute_analyzeThermodynamicSamplingPoints(simulation_id,
#                models_I=thermomodels,
#                data_dir_I=data_dir_I,
#                inconsistent_tcc_I=inconsistent_tcc,
#                remove_pointsNotInSolutionSpace_I=True,
#                min_pointsInSolutionSpace_I=1);
<commit_msg>updated main<commit_after>import sys
sys.path.append('C:/Users/dmccloskey-sbrg/Documents/GitHub/SBaaS_base')
from SBaaS_base.postgresql_settings import postgresql_settings
from SBaaS_base.postgresql_orm import postgresql_orm

# read in the settings file
filename = 'C:/Users/dmccloskey-sbrg/Google Drive/SBaaS_settings/settings_metabolomics.ini';
pg_settings = postgresql_settings(filename);

# connect to the database from the settings file
pg_orm = postgresql_orm();
pg_orm.set_sessionFromSettings(pg_settings.database_settings);
session = pg_orm.get_session();
engine = pg_orm.get_engine();

# your app...
# SBaaS paths:
sys.path.append(pg_settings.datadir_settings['github']+'/SBaaS_base')
sys.path.append(pg_settings.datadir_settings['github']+'/SBaaS_LIMS')
sys.path.append(pg_settings.datadir_settings['github']+'/SBaaS_quantification')
sys.path.append(pg_settings.datadir_settings['github']+'/SBaaS_physiology')
sys.path.append(pg_settings.datadir_settings['github']+'/SBaaS_MFA')
sys.path.append(pg_settings.datadir_settings['github']+'/SBaaS_visualization')
sys.path.append(pg_settings.datadir_settings['github']+'/SBaaS_models')
sys.path.append(pg_settings.datadir_settings['github']+'/SBaaS_thermodynamics')
sys.path.append(pg_settings.datadir_settings['github']+'/SBaaS_COBRA')
# SBaaS dependencies paths:
sys.path.append(pg_settings.datadir_settings['github']+'/io_utilities')
sys.path.append(pg_settings.datadir_settings['github']+'/quantification_analysis')
sys.path.append(pg_settings.datadir_settings['github']+'/matplotlib_utilities')
sys.path.append(pg_settings.datadir_settings['github']+'/thermodynamics/thermodynamics')
sys.path.append(pg_settings.datadir_settings['github']+'/component-contribution')
sys.path.append(pg_settings.datadir_settings['github']+'/molmass')
sys.path.append(pg_settings.datadir_settings['github']+'/python_statistics')
sys.path.append(pg_settings.datadir_settings['github']+'/r_statistics')
sys.path.append(pg_settings.datadir_settings['github']+'/listDict')
sys.path.append(pg_settings.datadir_settings['github']+'/ddt_python')

#make the measuredData table
from SBaaS_thermodynamics.stage03_quantification_measuredData_execute import stage03_quantification_measuredData_execute
exmeasuredData01 = stage03_quantification_measuredData_execute(session,engine,pg_settings.datadir_settings)
exmeasuredData01.initialize_dataStage03_quantification_measuredData();
exmeasuredData01.initialize_supportedTables();
exmeasuredData01.initialize_tables()

#make the COBRA table
from SBaaS_models.models_COBRA_execute import models_COBRA_execute
exCOBRA01 = models_COBRA_execute(session,engine,pg_settings.datadir_settings);
exCOBRA01.initialize_supportedTables();
exCOBRA01.initialize_tables()

##load '150526_iDM2015'
#exCOBRA01.import_dataStage02PhysiologyModel_json(
#    model_id_I='151026_iDM2015_irreversible',
#    date_I='2015-10-26 00:00:00',
#    model_json=pg_settings.datadir_settings['workspace_data']+ '/models/150526_iDM2015.json'
#    );

#pre-load the models
thermomodels = exCOBRA01.get_models(model_ids_I=["iJO1366_irreversible"]);

#make the measuredData table
from SBaaS_thermodynamics.stage03_quantification_measuredData_execute import stage03_quantification_measuredData_execute
exmeasuredData01 = stage03_quantification_measuredData_execute(session,engine,pg_settings.datadir_settings)
exmeasuredData01.initialize_supportedTables();
exmeasuredData01.initialize_tables()

#reset previous experimental data imports
exmeasuredData01.reset_dataStage03_quantification_metabolomicsData('IndustrialStrains03');

#transfer measured metabolomics data from data_stage01_quantification_averagesMIGeo
exmeasuredData01.execute_makeMetabolomicsData_intracellular('IndustrialStrains03');

#import exometabolomic information (i.e., media)
exmeasuredData01.import_dataStage03QuantificationMetabolomicsData_add(pg_settings.datadir_settings['workspace_data']+'/_input/141007_data_stage03_quantification_metabolomicsData_glcM902.csv');

#make the otherData table
from SBaaS_thermodynamics.stage03_quantification_otherData_execute import stage03_quantification_otherData_execute
exotherData01 = stage03_quantification_otherData_execute(session,engine,pg_settings.datadir_settings)
exdGr01.initialize_supportedTables();
exdGr01.initialize_tables()

#reset previous experimental data imports
exotherData01.reset_dataStage03_quantification_otherData('IndustrialStrains03');

# import the pH, ionic strength, and temperature for the simulation
exotherData01.import_dataStage03OtherData_add(pg_settings.datadir_settings['workspace_data']+'/_input/141007_data_stage03_quantification_otherData.csv');

#make the simulatedData table
from SBaaS_thermodynamics.stage03_quantification_simulatedData_execute import stage03_quantification_simulatedData_execute
exsimData01 = stage03_quantification_simulatedData_execute(session,engine,pg_settings.datadir_settings)
exsimData01.initialize_supportedTables();
exsimData01.initialize_tables()

#reset previous experiments
exsimData01.reset_dataStage03_quantification_simulatedData('IndustrialStrains03')

# perform FVA and single reaction deletion simulations
exsimData01.execute_makeSimulatedData('IndustrialStrains03',models_I=thermomodels)

#make the dG_f table
from SBaaS_thermodynamics.stage03_quantification_dG_f_execute import stage03_quantification_dG_f_execute
exdGf01 = stage03_quantification_dG_f_execute(session,engine,pg_settings.datadir_settings)
exdGf01.initialize_supportedTables();
exdGf01.initialize_tables()

#reset previous dG_f adjustments
exdGf01.reset_dataStage03_quantification_dG_f('IndustrialStrains03');

# adjust dG0 compound formation energies to the in vivo dG compound formation energies
# i.e, to the specified pH, ionic strength and temperature
exdGf01.execute_adjust_dG_f('IndustrialStrains03',models_I=thermomodels);

#make the dG_r table
from SBaaS_thermodynamics.stage03_quantification_dG_r_execute import stage03_quantification_dG_r_execute
exdGr01 = stage03_quantification_dG_r_execute(session,engine,pg_settings.datadir_settings)
exdGr01.initialize_supportedTables();
exdGr01.initialize_tables()

# reset previous analyses
exdGr01.reset_dataStage03_quantification_dG_r_all('IndustrialStrains03');

# calculate the in vivo dG reaction energies from adjusted dG_f and metabolomics values
# 1. dG0_r values are first calculated from the dG_f values
# 2. dG_r values are calculated from the dG0_r values and measured data
# 3. A thermodynamic consistency check is performed based on
#    FVA, SRA, and dG_r values
exdGr01.execute_calculate_dG_r('IndustrialStrains03',models_I=thermomodels);

exdGr01.reset_dataStage03_quantification_dG_r_comparison(
    analysis_id_I='ALEsKOs01_0_evo04_0_11_evo04gnd'
    );

#perform a thermodynamic comparison
exdGr01.execute_compare_dG_r(
    analysis_id_I='ALEsKOs01_0_evo04_0_11_evo04gnd',
    simulation_id_base_I='ALEsKOs01_iJO1366_OxicEvo04EcoliGlc_0',
    simulation_ids_I=[],
    models_I=thermomodels,
    measured_concentration_coverage_criteria_I=0.5,
    measured_dG_f_coverage_criteria_I=0.99)<|endoftext|>


<commit_before>import os
import vtkAll as vtk
import math
import time
import numpy as np

from ddapp import callbacks
from ddapp import transformUtils
from ddapp import lcmUtils
from ddapp import objectmodel as om
from ddapp.utime import getUtime
from ddapp import robotstate

import drc as lcmdrc

class ManipulationPlanItem(om.ObjectModelItem):
    pass

class ManipulationPlanDriver(object):

    PLAN_RECEIVED = 'PLAN_RECEIVED'
    PLAN_COMMITTED = 'PLAN_COMMITTED'

    def __init__(self):
        lcmUtils.addSubscriber('CANDIDATE_MANIP_PLAN', lcmdrc.robot_plan_w_keyframes_t, self.onManipPlan)
        lcmUtils.addSubscriber('CANDIDATE_ROBOT_PLAN_WITH_SUPPORTS',lcmdrc.robot_plan_with_supports_t, self.onPlanWithSupports)
        self.lastManipPlan = None
        self.committedPlans = []
        self.callbacks = callbacks.CallbackRegistry([self.PLAN_RECEIVED,
                                                     self.PLAN_COMMITTED])

    def onManipPlan(self, msg):
        self.lastManipPlan = msg
        self.callbacks.process(self.PLAN_RECEIVED, msg)

    def onPlanWithSupports(self, msg):
        self.lastManipPlan = msg #does it matter that the type of msg is different here? it containts a robot_plan_t message type
        self.callbacks.process(self.PLAN_RECEIVED, msg)

    def convertKeyframePlan(self, keyframeMsg):
        msg = lcmdrc.robot_plan_t()
        msg.utime = keyframeMsg.utime
        msg.robot_name = keyframeMsg.robot_name
        msg.num_states = keyframeMsg.num_states
        msg.plan = keyframeMsg.plan
        msg.plan_info = keyframeMsg.plan_info
        msg.num_bytes = keyframeMsg.num_bytes
        msg.matlab_data = keyframeMsg.matlab_data
        msg.num_grasp_transitions = keyframeMsg.num_grasp_transitions

        msg.left_arm_control_type = msg.NONE
        msg.right_arm_control_type = msg.NONE
        msg.left_leg_control_type = msg.NONE
        msg.right_leg_control_type = msg.NONE

        return msg

    def commitManipPlan(self, manipPlan):

        if manipPlan in self.committedPlans:
            raise Exception('Refusing to re-commit manipulation plan.')
        self.committedPlans.append(manipPlan)

        if isinstance(manipPlan, lcmdrc.robot_plan_w_keyframes_t):
            manipPlan = self.convertKeyframePlan(manipPlan)
        manipPlan.utime = getUtime()

        channelMap = {lcmdrc.robot_plan_with_supports_t:'COMMITTED_ROBOT_PLAN_WITH_SUPPORTS'}
        defaultChannel = 'COMMITTED_ROBOT_PLAN'
        channel = channelMap.get(type(manipPlan), defaultChannel)
        lcmUtils.publish(channel, manipPlan)
        self.callbacks.process(self.PLAN_COMMITTED, manipPlan)

    def sendPlanPause(self):
        msg = lcmdrc.plan_control_t()
        msg.utime = getUtime()
        msg.control = lcmdrc.plan_control_t.PAUSE
        lcmUtils.publish('COMMITTED_PLAN_PAUSE', msg)

    def connectPlanReceived(self, func):
        return self.callbacks.connect(self.PLAN_RECEIVED, func)

    def disconnectPlanReceived(self, callbackId):
        self.callbacks.disconnect(callbackId)

    def connectPlanCommitted(self, func):
        return self.callbacks.connect(self.PLAN_COMMITTED, func)

    def disconnectPlanCommitted(self, callbackId):
        self.callbacks.disconnect(callbackId)

<commit_msg>remove unnecessary function<commit_after>import os
import vtkAll as vtk
import math
import time
import numpy as np

from ddapp import callbacks
from ddapp import transformUtils
from ddapp import lcmUtils
from ddapp import objectmodel as om
from ddapp.utime import getUtime
from ddapp import robotstate

import drc as lcmdrc

class ManipulationPlanItem(om.ObjectModelItem):
    pass

class ManipulationPlanDriver(object):

    PLAN_RECEIVED = 'PLAN_RECEIVED'
    PLAN_COMMITTED = 'PLAN_COMMITTED'

    def __init__(self):
        lcmUtils.addSubscriber('CANDIDATE_MANIP_PLAN', lcmdrc.robot_plan_w_keyframes_t, self.onManipPlan)
        lcmUtils.addSubscriber('CANDIDATE_ROBOT_PLAN_WITH_SUPPORTS',lcmdrc.robot_plan_with_supports_t, self.onManipPlan)
        self.lastManipPlan = None
        self.committedPlans = []
        self.callbacks = callbacks.CallbackRegistry([self.PLAN_RECEIVED,
                                                     self.PLAN_COMMITTED])

    def onManipPlan(self, msg):
        self.lastManipPlan = msg
        self.callbacks.process(self.PLAN_RECEIVED, msg)

    def convertKeyframePlan(self, keyframeMsg):
        msg = lcmdrc.robot_plan_t()
        msg.utime = keyframeMsg.utime
        msg.robot_name = keyframeMsg.robot_name
        msg.num_states = keyframeMsg.num_states
        msg.plan = keyframeMsg.plan
        msg.plan_info = keyframeMsg.plan_info
        msg.num_bytes = keyframeMsg.num_bytes
        msg.matlab_data = keyframeMsg.matlab_data
        msg.num_grasp_transitions = keyframeMsg.num_grasp_transitions

        msg.left_arm_control_type = msg.NONE
        msg.right_arm_control_type = msg.NONE
        msg.left_leg_control_type = msg.NONE
        msg.right_leg_control_type = msg.NONE

        return msg

    def commitManipPlan(self, manipPlan):

        if manipPlan in self.committedPlans:
            raise Exception('Refusing to re-commit manipulation plan.')
        self.committedPlans.append(manipPlan)

        if isinstance(manipPlan, lcmdrc.robot_plan_w_keyframes_t):
            manipPlan = self.convertKeyframePlan(manipPlan)
        manipPlan.utime = getUtime()

        channelMap = {lcmdrc.robot_plan_with_supports_t:'COMMITTED_ROBOT_PLAN_WITH_SUPPORTS'}
        defaultChannel = 'COMMITTED_ROBOT_PLAN'
        channel = channelMap.get(type(manipPlan), defaultChannel)
        lcmUtils.publish(channel, manipPlan)
        self.callbacks.process(self.PLAN_COMMITTED, manipPlan)

    def sendPlanPause(self):
        msg = lcmdrc.plan_control_t()
        msg.utime = getUtime()
        msg.control = lcmdrc.plan_control_t.PAUSE
        lcmUtils.publish('COMMITTED_PLAN_PAUSE', msg)

    def connectPlanReceived(self, func):
        return self.callbacks.connect(self.PLAN_RECEIVED, func)

    def disconnectPlanReceived(self, callbackId):
        self.callbacks.disconnect(callbackId)

    def connectPlanCommitted(self, func):
        return self.callbacks.connect(self.PLAN_COMMITTED, func)

    def disconnectPlanCommitted(self, callbackId):
        self.callbacks.disconnect(callbackId)

<|endoftext|>