<commit_before># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pecan
import six

from mongoengine import ValidationError
from st2api.controllers import resource
from st2common import log as logging
from st2common.exceptions.actionalias import ActionAliasAmbiguityException
from st2common.exceptions.apivalidation import ValueValidationException
from st2common.models.api.action import ActionAliasAPI
from st2common.persistence.actionalias import ActionAlias
from st2common.models.api.base import jsexpose
from st2common.rbac.types import PermissionType
from st2common.rbac.decorators import request_user_has_permission
from st2common.rbac.decorators import request_user_has_resource_api_permission
from st2common.rbac.decorators import request_user_has_resource_db_permission

from st2common.util.actionalias_matching import match_command_to_alias

http_client = six.moves.http_client

LOG = logging.getLogger(__name__)

class ActionAliasController(resource.ContentPackResourceController):
    """
        Implements the RESTful interface for ActionAliases.
    """
    model = ActionAliasAPI
    access = ActionAlias
    supported_filters = {
        'name': 'name',
        'pack': 'pack'
    }

    query_options = {
        'sort': ['pack', 'name']
    }

    _custom_actions = {
        'match': ['POST']
    }

    @request_user_has_permission(permission_type=PermissionType.ACTION_ALIAS_LIST)
    @jsexpose()
    def get_all(self, **kwargs):
        return super(ActionAliasController, self)._get_all(**kwargs)

    @request_user_has_resource_db_permission(permission_type=PermissionType.ACTION_ALIAS_VIEW)
    @jsexpose(arg_types=[str])
    def get_one(self, ref_or_id):
        return super(ActionAliasController, self)._get_one(ref_or_id)

    @jsexpose(arg_types=[str], status_code=http_client.ACCEPTED)
    def match(self, command, **kwargs):
        """
            Run a chatops command

            Handles requests:
                POST /chatops/match

                command=hello%20world
        """
        try:
            # 1. Get aliases
            aliases = super(ActionAliasController, self)._get_all(**kwargs)
            # 2. Match alias(es) to command
            matches = match_command_to_alias(command, aliases)
            if len(matches) > 1:
                raise ActionAliasAmbiguityException("Command matched more than 1 pattern",
                                                    matches=matches)
            return matches
        except (ActionAliasAmbiguityException) as e:
            # TODO : error on unmatched alias
            LOG.exception('Command matched (%s) patterns.', len(e.matches))
            pecan.abort(http_client.BAD_REQUEST, str(e))
            return e.matches

    @jsexpose(body_cls=ActionAliasAPI, status_code=http_client.CREATED)
    @request_user_has_resource_api_permission(permission_type=PermissionType.ACTION_ALIAS_CREATE)
    def post(self, action_alias):
        """
            Create a new ActionAlias.

            Handles requests:
                POST /actionalias/
        """
        try:
            action_alias_db = ActionAliasAPI.to_model(action_alias)
            LOG.debug('/actionalias/ POST verified ActionAliasAPI and formulated ActionAliasDB=%s',
                      action_alias_db)
            action_alias_db = ActionAlias.add_or_update(action_alias_db)
        except (ValidationError, ValueError, ValueValidationException) as e:
            LOG.exception('Validation failed for action alias data=%s.', action_alias)
            pecan.abort(http_client.BAD_REQUEST, str(e))
            return

        extra = {'action_alias_db': action_alias_db}
        LOG.audit('Action alias created. ActionAlias.id=%s' % (action_alias_db.id), extra=extra)
        action_alias_api = ActionAliasAPI.from_model(action_alias_db)

        return action_alias_api

    @request_user_has_resource_db_permission(permission_type=PermissionType.ACTION_MODIFY)
    @jsexpose(arg_types=[str], body_cls=ActionAliasAPI)
    def put(self, action_alias_ref_or_id, action_alias):
        action_alias_db = self._get_by_ref_or_id(ref_or_id=action_alias_ref_or_id)
        LOG.debug('PUT /actionalias/ lookup with id=%s found object: %s', action_alias_ref_or_id,
                  action_alias_db)

        try:
            if action_alias.id is not None and action_alias.id is not '' and \
               action_alias.id != action_alias_ref_or_id:
                LOG.warning('Discarding mismatched id=%s found in payload and using uri_id=%s.',
                            action_alias.id, action_alias_ref_or_id)
            old_action_alias_db = action_alias_db
            action_alias_db = ActionAliasAPI.to_model(action_alias)
            action_alias_db.id = action_alias_ref_or_id
            action_alias_db = ActionAlias.add_or_update(action_alias_db)
        except (ValidationError, ValueError) as e:
            LOG.exception('Validation failed for action alias data=%s', action_alias)
            pecan.abort(http_client.BAD_REQUEST, str(e))
            return

        extra = {'old_action_alias_db': old_action_alias_db, 'new_action_alias_db': action_alias_db}
        LOG.audit('Action alias updated. ActionAlias.id=%s.' % (action_alias_db.id), extra=extra)
        action_alias_api = ActionAliasAPI.from_model(action_alias_db)

        return action_alias_api

    @request_user_has_resource_db_permission(permission_type=PermissionType.ACTION_ALIAS_DELETE)
    @jsexpose(arg_types=[str], status_code=http_client.NO_CONTENT)
    def delete(self, action_alias_ref_or_id):
        """
            Delete an action alias.

            Handles requests:
                DELETE /actionalias/1
        """
        action_alias_db = self._get_by_ref_or_id(ref_or_id=action_alias_ref_or_id)
        LOG.debug('DELETE /actionalias/ lookup with id=%s found object: %s', action_alias_ref_or_id,
                  action_alias_db)
        try:
            ActionAlias.delete(action_alias_db)
        except Exception as e:
            LOG.exception('Database delete encountered exception during delete of id="%s".',
                          action_alias_ref_or_id)
            pecan.abort(http_client.INTERNAL_SERVER_ERROR, str(e))
            return

        extra = {'action_alias_db': action_alias_db}
        LOG.audit('Action alias deleted. ActionAlias.id=%s.' % (action_alias_db.id), extra=extra)
<commit_msg>return a dict instead of a list of tuples<commit_after># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pecan
import six

from mongoengine import ValidationError
from st2api.controllers import resource
from st2common import log as logging
from st2common.exceptions.actionalias import ActionAliasAmbiguityException
from st2common.exceptions.apivalidation import ValueValidationException
from st2common.models.api.action import ActionAliasAPI
from st2common.persistence.actionalias import ActionAlias
from st2common.models.api.base import jsexpose
from st2common.rbac.types import PermissionType
from st2common.rbac.decorators import request_user_has_permission
from st2common.rbac.decorators import request_user_has_resource_api_permission
from st2common.rbac.decorators import request_user_has_resource_db_permission

from st2common.util.actionalias_matching import match_command_to_alias

http_client = six.moves.http_client

LOG = logging.getLogger(__name__)

class ActionAliasController(resource.ContentPackResourceController):
    """
        Implements the RESTful interface for ActionAliases.
    """
    model = ActionAliasAPI
    access = ActionAlias
    supported_filters = {
        'name': 'name',
        'pack': 'pack'
    }

    query_options = {
        'sort': ['pack', 'name']
    }

    _custom_actions = {
        'match': ['POST']
    }

    def _match_tuple_to_dict(self, match):
        return {
            'actionalias': match[0],
            'display': match[1],
            'representation': match[2]
        }

    @request_user_has_permission(permission_type=PermissionType.ACTION_ALIAS_LIST)
    @jsexpose()
    def get_all(self, **kwargs):
        return super(ActionAliasController, self)._get_all(**kwargs)

    @request_user_has_resource_db_permission(permission_type=PermissionType.ACTION_ALIAS_VIEW)
    @jsexpose(arg_types=[str])
    def get_one(self, ref_or_id):
        return super(ActionAliasController, self)._get_one(ref_or_id)

    @jsexpose(arg_types=[str], status_code=http_client.ACCEPTED)
    def match(self, command, **kwargs):
        """
            Run a chatops command

            Handles requests:
                POST /chatops/match

                command=hello%20world
        """
        try:
            # 1. Get aliases
            aliases = super(ActionAliasController, self)._get_all(**kwargs)
            # 2. Match alias(es) to command
            matches = match_command_to_alias(command, aliases)
            if len(matches) > 1:
                raise ActionAliasAmbiguityException("Command matched more than 1 pattern",
                                                    matches=matches)
            return [self._match_tuple_to_dict(match) for match in matches]
        except (ActionAliasAmbiguityException) as e:
            # TODO : error on unmatched alias
            LOG.exception('Command matched (%s) patterns.', len(e.matches))
            pecan.abort(http_client.BAD_REQUEST, str(e))
            return [self._match_tuple_to_dict(match) for match in e.matches]

    @jsexpose(body_cls=ActionAliasAPI, status_code=http_client.CREATED)
    @request_user_has_resource_api_permission(permission_type=PermissionType.ACTION_ALIAS_CREATE)
    def post(self, action_alias):
        """
            Create a new ActionAlias.

            Handles requests:
                POST /actionalias/
        """
        try:
            action_alias_db = ActionAliasAPI.to_model(action_alias)
            LOG.debug('/actionalias/ POST verified ActionAliasAPI and formulated ActionAliasDB=%s',
                      action_alias_db)
            action_alias_db = ActionAlias.add_or_update(action_alias_db)
        except (ValidationError, ValueError, ValueValidationException) as e:
            LOG.exception('Validation failed for action alias data=%s.', action_alias)
            pecan.abort(http_client.BAD_REQUEST, str(e))
            return

        extra = {'action_alias_db': action_alias_db}
        LOG.audit('Action alias created. ActionAlias.id=%s' % (action_alias_db.id), extra=extra)
        action_alias_api = ActionAliasAPI.from_model(action_alias_db)

        return action_alias_api

    @request_user_has_resource_db_permission(permission_type=PermissionType.ACTION_MODIFY)
    @jsexpose(arg_types=[str], body_cls=ActionAliasAPI)
    def put(self, action_alias_ref_or_id, action_alias):
        action_alias_db = self._get_by_ref_or_id(ref_or_id=action_alias_ref_or_id)
        LOG.debug('PUT /actionalias/ lookup with id=%s found object: %s', action_alias_ref_or_id,
                  action_alias_db)

        try:
            if action_alias.id is not None and action_alias.id is not '' and \
               action_alias.id != action_alias_ref_or_id:
                LOG.warning('Discarding mismatched id=%s found in payload and using uri_id=%s.',
                            action_alias.id, action_alias_ref_or_id)
            old_action_alias_db = action_alias_db
            action_alias_db = ActionAliasAPI.to_model(action_alias)
            action_alias_db.id = action_alias_ref_or_id
            action_alias_db = ActionAlias.add_or_update(action_alias_db)
        except (ValidationError, ValueError) as e:
            LOG.exception('Validation failed for action alias data=%s', action_alias)
            pecan.abort(http_client.BAD_REQUEST, str(e))
            return

        extra = {'old_action_alias_db': old_action_alias_db, 'new_action_alias_db': action_alias_db}
        LOG.audit('Action alias updated. ActionAlias.id=%s.' % (action_alias_db.id), extra=extra)
        action_alias_api = ActionAliasAPI.from_model(action_alias_db)

        return action_alias_api

    @request_user_has_resource_db_permission(permission_type=PermissionType.ACTION_ALIAS_DELETE)
    @jsexpose(arg_types=[str], status_code=http_client.NO_CONTENT)
    def delete(self, action_alias_ref_or_id):
        """
            Delete an action alias.

            Handles requests:
                DELETE /actionalias/1
        """
        action_alias_db = self._get_by_ref_or_id(ref_or_id=action_alias_ref_or_id)
        LOG.debug('DELETE /actionalias/ lookup with id=%s found object: %s', action_alias_ref_or_id,
                  action_alias_db)
        try:
            ActionAlias.delete(action_alias_db)
        except Exception as e:
            LOG.exception('Database delete encountered exception during delete of id="%s".',
                          action_alias_ref_or_id)
            pecan.abort(http_client.INTERNAL_SERVER_ERROR, str(e))
            return

        extra = {'action_alias_db': action_alias_db}
        LOG.audit('Action alias deleted. ActionAlias.id=%s.' % (action_alias_db.id), extra=extra)
<|endoftext|>


<commit_before>'''
Created on Jan 27, 2011

@author: peyman kazemian
'''
from sts.headerspace.headerspace.hs import *
from sts.headerspace.headerspace.tf import *
from sts.headerspace.config_parser.openflow_parser import get_uniq_port_id
import sts.headerspace.config_parser.openflow_parser as of

import sys
import glob
import os
import subprocess

# What is a p_node?
# A hash apparently:
#  hdr -> foo
#  port -> foo
#  visits -> foo
def print_p_node(p_node):
    print "-----"
    print p_node["hdr"]
    print p_node["port"]
    print p_node["visits"]
    print "-----"

def find_reachability(NTF, TTF, in_port, out_ports, input_pkt):
    paths = []
    propagation = []

    p_node = {}
    p_node["hdr"] = input_pkt
    p_node["port"] = in_port
    p_node["visits"] = []
    p_node["hs_history"] = []
    propagation.append(p_node)

    while len(propagation)>0:
        #get the next node in propagation graph and apply it to NTF and TTF
        print "Propagation has length: %d"%len(propagation)
        tmp_propagate = []
        for p_node in propagation:
            next_hp = NTF.T(p_node["hdr"],p_node["port"])
            for (next_h,next_ps) in next_hp:
                for next_p in next_ps:
                    if next_p in out_ports:
                        reached = {}
                        reached["hdr"] = next_h
                        reached["port"] = next_p
                        reached["visits"] = list(p_node["visits"])
                        reached["visits"].append(p_node["port"])
                        reached["hs_history"] = list(p_node["hs_history"])
                        paths.append(reached)
                    else:
                        linked = TTF.T(next_h,next_p)
                        for (linked_h,linked_ports) in linked:
                            for linked_p in linked_ports:
                                new_p_node = {}
                                new_p_node["hdr"] = linked_h
                                new_p_node["port"] = linked_p
                                new_p_node["visits"] = list(p_node["visits"])
                                new_p_node["visits"].append(p_node["port"])
                                new_p_node["visits"].append(next_p)
                                new_p_node["hs_history"] = list(p_node["hs_history"])
                                new_p_node["hs_history"].append(p_node["hdr"])
                                if linked_p in out_ports:
                                    paths.append(new_p_node)
                                elif linked_p in new_p_node["visits"]:
                                    print "WARNING: detected a loop - branch aborted: \nHeaderSpace: %s\n Visited Ports: %s\nLast Port %d "%(\
                                        new_p_node["hdr"],new_p_node["visits"],new_p_node["port"])
                                else:
                                    tmp_propagate.append(new_p_node)
        propagation = tmp_propagate

    return paths

def get_all_x(NTF):
  all_x = byte_array_get_all_x(NTF.length)
  test_pkt = headerspace(NTF.format)
  test_pkt.add_hs(all_x)
  return test_pkt

def detect_loop(NTF, TTF, ports, reverse_map, test_packet = None):
    loops = []
    for port in ports:
        print "port %d is being checked"%port
        propagation = []

        # put all-x test packet in propagation graph
        test_pkt = test_packet
        if test_pkt == None:
          test_pkt = get_all_x(NTF)

        p_node = {}
        p_node["hdr"] = test_pkt
        p_node["port"] = port
        p_node["visits"] = []
        p_node["hs_history"] = []

        propagation.append(p_node)
        while len(propagation)>0:
            #get the next node in propagation graph and apply it to NTF and TTF
            print "Propagation has length: %d"%len(propagation)
            tmp_propag = []
            for p_node in propagation:
                # hp is "header port"
                next_hp = NTF.T(p_node["hdr"],p_node["port"])
                for (next_h,next_ps) in next_hp:
                    for next_p in next_ps:
                        linked = TTF.T(next_h,next_p)
                        for (linked_h,linked_ports) in linked:
                            for linked_p in linked_ports:
                                new_p_node = {}
                                new_p_node["hdr"] = linked_h
                                new_p_node["port"] = linked_p
                                new_p_node["visits"] = list(p_node["visits"])
                                new_p_node["visits"].append(p_node["port"])
                                #new_p_node["visits"].append(next_p)
                                new_p_node["hs_history"] = list(p_node["hs_history"])
                                new_p_node["hs_history"].append(p_node["hdr"])
                                #print new_p_node
                                if len(new_p_node["visits"]) > 0 and new_p_node["visits"][0] == linked_p:
                                    loops.append(new_p_node)
                                    print "loop detected"
                                elif linked_p in new_p_node["visits"]:
#                                    if (linked_p not in ports):
#                                        print "WARNING: detected a loop whose port is not in checked ports - branch aborted:"
#                                        print_loops([new_p_node],reverse_map)
#                                        print "END OF WARNING"
                                    pass
                                else:
                                    tmp_propag.append(new_p_node)
            propagation = tmp_propag
    return loops

# TODO(cs): make this a parameter
# TODO(cs): don't assume that cwd is the sts top directory
HASSEL_C_PATH = "./sts/headerspace/hassel-c"
HASSEL_TF_PATH = HASSEL_C_PATH + "/tfs/sts"

# Omega defines the externally visible behavior of the network. Defined as a table:
#   (header space, edge_port) -> [(header_space, final_location),(header_space, final_location)...]
def compute_omega(name_tf_pairs, TTF, edge_links):
  omega = {}

  # Nuke old TF object files
  old_tfs = glob.glob(HASSEL_TF_PATH + "/*tf")
  for path in old_tfs:
    os.unlink(path)

  # Write out TF for each switch, and TTF to object files
  for name, tf in name_tf_pairs:
    tf.save_object_to_file(HASSEL_TF_PATH + "/" + name + ".tf")
  TTF.save_object_to_file(HASSEL_TF_PATH + "/topology.tf")

  # Generate the .dat file
  # Make sure we're in the right cwd
  old_cwd = os.getcwd()
  try:
    os.chdir(HASSEL_C_PATH)
    os.system("./gen sts")

    # TODO(cs): need to model host end of link, or does switch end suffice?
    edge_ports = map(lambda access_link: get_uniq_port_id(access_link.switch, access_link.switch_port), edge_links)
    print "edge_ports: %s" % edge_ports

    for start_port in edge_ports:
      port_omega = compute_single_omega(start_port, edge_ports)
      omega = dict(omega.items() + port_omega.items())
  finally:
    os.chdir(old_cwd)
  return omega

def compute_single_omega(start_port, edge_ports):
    if type(start_port) != int:
      start_port = get_uniq_port_id(start_port.switch, start_port.switch_port)

    if type(edge_ports) != list:
      edge_ports = list(edge_ports)
    if type(edge_ports[0]) != int:
      edge_ports = map(lambda access_link: get_uniq_port_id(access_link.switch, access_link.switch_port), edge_ports)

    print "port %d is being checked" % start_port

    str_start_port = str(start_port)
    str_edge_ports = map(str, edge_ports)
    proc = subprocess.Popen(["./sts", str(start_port)] + str_edge_ports,
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    omega = { start_port : [] }
    second_to_last_line = ''
    last_line = ''
    while True:
      # Format is:

      #  <Original> Port: ...
      #  <Next> Port: ...
      #  ...
      #  <Edge> Port: ...
      #  HS: D0,D0,D0,D8,D123,D123,...
      # -----
      #  <Original> Port -> ...
      #  ...

      line = proc.stdout.readline()
      if line == '':
        break

      if line.startswith("-----"):
        hs_string = last_line.split(":")[1].strip()
        port_stripped = second_to_last_line.split(":")[1].lstrip()
        out_port_string = port_stripped[0:port_stripped.index(",")]
        out_port = int(out_port_string)
        # TODO(cs): debug the code below to get human readable headerspace
        # strings
        #readable_hs = headerspace(of.hs_format)
        #readable_hs.add_hs(hs_string_to_byte_array(hs_string))
        #omega[start_port].append((readable_hs, out_port))
        omega[start_port].append((hs_string, out_port))

      second_to_last_line = last_line
      last_line = line

    return omega

def print_reachability(paths, reverse_map):
    for p_node in paths:
        str = ""
        for port in p_node["visits"]:
            if str == "":
                str = reverse_map["%d"%port]
            else:
                str = "%s ---> %s"%(str,reverse_map["%d"%port])
        str = "%s ---> %s"%(str,reverse_map["%d"%p_node["port"]])
        print "Path: %s"%str
        print "HS Received: %s"%p_node["hdr"]
        print "----------------------------------------------"

def print_loops(loops, reverse_map):
    for p_node in loops:
        print "----------------------------------------------"
        str = ""
        for port in p_node["visits"]:
            if str == "":
                str = reverse_map["%d"%port]
            else:
                str = "%s ---> %s"%(str,reverse_map["%d"%port])
        str = "%s ---> %s"%(str,reverse_map["%d"%p_node["port"]])
        print "Path: %s"%str
        rl_id =  "applied rules: "
        for (n,r,s) in p_node["hdr"].applied_rule_ids:
            rl_id = rl_id + " -> %s"%r
        print rl_id
        for i in range(len(p_node["hs_history"])):
            print "*** %d) AT PORT: %s\nHS: %s\n"%(i,reverse_map["%d"%p_node["visits"][i]],p_node["hs_history"][i])
        print "*** %d) AT PORT: %s\nHS: %s\n"%(i+1,reverse_map["%d"%p_node["port"]],p_node["hdr"])
        print "----------------------------------------------"

def loop_path_to_str(p_node, reverse_map):
    str = ""
    for port in p_node["visits"]:
        if str == "":
            str = reverse_map["%d"%port]
        else:
            str = "%s ---> %s"%(str,reverse_map["%d"%port])
    str = "%s ---> %s"%(str,reverse_map["%d"%p_node["port"]])
    return str

def trace_hs_back(applied_rule_list,hs,last_port):
    port = last_port
    hs_list = [hs]
    hs.applied_rule_ids = []
    for (tf,rule_id,in_port) in reversed(applied_rule_list):
        tmp = []
        for next_hs in hs_list:
            hp = tf.T_inv_rule(rule_id,next_hs,port)
            for (h,p_list) in hp:
                if in_port in p_list:
                    tmp.append(h)
        if tmp == []:
            break
        hs_list = tmp
        port = in_port
    return hs_list

def find_loop_original_header(NTF,TTF,propagation_node):
    applied_rule_ids = list(propagation_node["hdr"].applied_rule_ids)
    hs_list = trace_hs_back(applied_rule_ids,propagation_node["hdr"],propagation_node["port"])
    return hs_list

#def is_loop_infinite(NTF, TTF, propagation_node):
#    p_node = propagation_node
#    header_cpy = p_node["hdr"].copy()
#    while len(p_node['visits']) > 0:
#        cur_port = p_node['visits'].pop()
#
#         transfer back on the link
#        linked = TTF.T_inv(p_node["hdr"],p_node["port"])
#
#         transfer back on node
#        prev_hp = NTF.T_inv(linked[0][0],linked[0][1])
#
#        prev_hs = headerspace(NTF.length)
#        for (h,p) in prev_hp:
#            if p == cur_port:
#                prev_hs.add_hs(h)
#
#        p_node["hdr"] = prev_hs
#        p_node["port"] = cur_port

    #print "hret = %s and"%propagation_node["hdr"]
    #print "hret_inv = %s"%(p_node["hdr"])
#    if propagation_node["hdr"].is_subset_of(p_node["hdr"]):
#        return (True,p_node["hdr"])
#    else:
#        insect = propagation_node["hdr"].copy_intersect(p_node["hdr"])
#        if insect.count() == 0:
#            return (False,[])
#        else:
#            new_cpy = propagation_node.copy()
#            new_cpy["hdr"] = insect
#            return is_loop_infinite(NTF, TTF, new_cpy)

<commit_msg>Adding runtime check for whether hassel-c has been complied<commit_after>'''
Created on Jan 27, 2011

@author: peyman kazemian
'''
from sts.headerspace.headerspace.hs import *
from sts.headerspace.headerspace.tf import *
from sts.headerspace.config_parser.openflow_parser import get_uniq_port_id
import sts.headerspace.config_parser.openflow_parser as of

import sys
import glob
import os
import subprocess

# What is a p_node?
# A hash apparently:
#  hdr -> foo
#  port -> foo
#  visits -> foo
def print_p_node(p_node):
    print "-----"
    print p_node["hdr"]
    print p_node["port"]
    print p_node["visits"]
    print "-----"

def find_reachability(NTF, TTF, in_port, out_ports, input_pkt):
    paths = []
    propagation = []

    p_node = {}
    p_node["hdr"] = input_pkt
    p_node["port"] = in_port
    p_node["visits"] = []
    p_node["hs_history"] = []
    propagation.append(p_node)

    while len(propagation)>0:
        #get the next node in propagation graph and apply it to NTF and TTF
        print "Propagation has length: %d"%len(propagation)
        tmp_propagate = []
        for p_node in propagation:
            next_hp = NTF.T(p_node["hdr"],p_node["port"])
            for (next_h,next_ps) in next_hp:
                for next_p in next_ps:
                    if next_p in out_ports:
                        reached = {}
                        reached["hdr"] = next_h
                        reached["port"] = next_p
                        reached["visits"] = list(p_node["visits"])
                        reached["visits"].append(p_node["port"])
                        reached["hs_history"] = list(p_node["hs_history"])
                        paths.append(reached)
                    else:
                        linked = TTF.T(next_h,next_p)
                        for (linked_h,linked_ports) in linked:
                            for linked_p in linked_ports:
                                new_p_node = {}
                                new_p_node["hdr"] = linked_h
                                new_p_node["port"] = linked_p
                                new_p_node["visits"] = list(p_node["visits"])
                                new_p_node["visits"].append(p_node["port"])
                                new_p_node["visits"].append(next_p)
                                new_p_node["hs_history"] = list(p_node["hs_history"])
                                new_p_node["hs_history"].append(p_node["hdr"])
                                if linked_p in out_ports:
                                    paths.append(new_p_node)
                                elif linked_p in new_p_node["visits"]:
                                    print "WARNING: detected a loop - branch aborted: \nHeaderSpace: %s\n Visited Ports: %s\nLast Port %d "%(\
                                        new_p_node["hdr"],new_p_node["visits"],new_p_node["port"])
                                else:
                                    tmp_propagate.append(new_p_node)
        propagation = tmp_propagate

    return paths

def get_all_x(NTF):
  all_x = byte_array_get_all_x(NTF.length)
  test_pkt = headerspace(NTF.format)
  test_pkt.add_hs(all_x)
  return test_pkt

def detect_loop(NTF, TTF, ports, reverse_map, test_packet = None):
    loops = []
    for port in ports:
        print "port %d is being checked"%port
        propagation = []

        # put all-x test packet in propagation graph
        test_pkt = test_packet
        if test_pkt == None:
          test_pkt = get_all_x(NTF)

        p_node = {}
        p_node["hdr"] = test_pkt
        p_node["port"] = port
        p_node["visits"] = []
        p_node["hs_history"] = []

        propagation.append(p_node)
        while len(propagation)>0:
            #get the next node in propagation graph and apply it to NTF and TTF
            print "Propagation has length: %d"%len(propagation)
            tmp_propag = []
            for p_node in propagation:
                # hp is "header port"
                next_hp = NTF.T(p_node["hdr"],p_node["port"])
                for (next_h,next_ps) in next_hp:
                    for next_p in next_ps:
                        linked = TTF.T(next_h,next_p)
                        for (linked_h,linked_ports) in linked:
                            for linked_p in linked_ports:
                                new_p_node = {}
                                new_p_node["hdr"] = linked_h
                                new_p_node["port"] = linked_p
                                new_p_node["visits"] = list(p_node["visits"])
                                new_p_node["visits"].append(p_node["port"])
                                #new_p_node["visits"].append(next_p)
                                new_p_node["hs_history"] = list(p_node["hs_history"])
                                new_p_node["hs_history"].append(p_node["hdr"])
                                #print new_p_node
                                if len(new_p_node["visits"]) > 0 and new_p_node["visits"][0] == linked_p:
                                    loops.append(new_p_node)
                                    print "loop detected"
                                elif linked_p in new_p_node["visits"]:
#                                    if (linked_p not in ports):
#                                        print "WARNING: detected a loop whose port is not in checked ports - branch aborted:"
#                                        print_loops([new_p_node],reverse_map)
#                                        print "END OF WARNING"
                                    pass
                                else:
                                    tmp_propag.append(new_p_node)
            propagation = tmp_propag
    return loops

# TODO(cs): make this a parameter
# TODO(cs): don't assume that cwd is the sts top directory
HASSEL_C_PATH = "./sts/headerspace/hassel-c"
HASSEL_TF_PATH = HASSEL_C_PATH + "/tfs/sts"

# Omega defines the externally visible behavior of the network. Defined as a table:
#   (header space, edge_port) -> [(header_space, final_location),(header_space, final_location)...]
def compute_omega(name_tf_pairs, TTF, edge_links):
  if not os.path.exists(HASSEL_C_PATH + "/gen"):
    raise RuntimeError("You need to make hassel-c!")
  omega = {}

  # Nuke old TF object files
  old_tfs = glob.glob(HASSEL_TF_PATH + "/*tf")
  for path in old_tfs:
    os.unlink(path)

  # Write out TF for each switch, and TTF to object files
  for name, tf in name_tf_pairs:
    tf.save_object_to_file(HASSEL_TF_PATH + "/" + name + ".tf")
  TTF.save_object_to_file(HASSEL_TF_PATH + "/topology.tf")

  # Generate the .dat file
  # Make sure we're in the right cwd
  old_cwd = os.getcwd()
  try:
    os.chdir(HASSEL_C_PATH)
    os.system("./gen sts")

    # TODO(cs): need to model host end of link, or does switch end suffice?
    edge_ports = map(lambda access_link: get_uniq_port_id(access_link.switch, access_link.switch_port), edge_links)
    print "edge_ports: %s" % edge_ports

    for start_port in edge_ports:
      port_omega = compute_single_omega(start_port, edge_ports)
      omega = dict(omega.items() + port_omega.items())
  finally:
    os.chdir(old_cwd)
  return omega

def compute_single_omega(start_port, edge_ports):
    if type(start_port) != int:
      start_port = get_uniq_port_id(start_port.switch, start_port.switch_port)

    if type(edge_ports) != list:
      edge_ports = list(edge_ports)
    if type(edge_ports[0]) != int:
      edge_ports = map(lambda access_link: get_uniq_port_id(access_link.switch, access_link.switch_port), edge_ports)

    print "port %d is being checked" % start_port

    str_start_port = str(start_port)
    str_edge_ports = map(str, edge_ports)
    proc = subprocess.Popen(["./sts", str(start_port)] + str_edge_ports,
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    omega = { start_port : [] }
    second_to_last_line = ''
    last_line = ''
    while True:
      # Format is:

      #  <Original> Port: ...
      #  <Next> Port: ...
      #  ...
      #  <Edge> Port: ...
      #  HS: D0,D0,D0,D8,D123,D123,...
      # -----
      #  <Original> Port -> ...
      #  ...

      line = proc.stdout.readline()
      if line == '':
        break

      if line.startswith("-----"):
        hs_string = last_line.split(":")[1].strip()
        port_stripped = second_to_last_line.split(":")[1].lstrip()
        out_port_string = port_stripped[0:port_stripped.index(",")]
        out_port = int(out_port_string)
        # TODO(cs): debug the code below to get human readable headerspace
        # strings
        #readable_hs = headerspace(of.hs_format)
        #readable_hs.add_hs(hs_string_to_byte_array(hs_string))
        #omega[start_port].append((readable_hs, out_port))
        omega[start_port].append((hs_string, out_port))

      second_to_last_line = last_line
      last_line = line

    return omega

def print_reachability(paths, reverse_map):
    for p_node in paths:
        str = ""
        for port in p_node["visits"]:
            if str == "":
                str = reverse_map["%d"%port]
            else:
                str = "%s ---> %s"%(str,reverse_map["%d"%port])
        str = "%s ---> %s"%(str,reverse_map["%d"%p_node["port"]])
        print "Path: %s"%str
        print "HS Received: %s"%p_node["hdr"]
        print "----------------------------------------------"

def print_loops(loops, reverse_map):
    for p_node in loops:
        print "----------------------------------------------"
        str = ""
        for port in p_node["visits"]:
            if str == "":
                str = reverse_map["%d"%port]
            else:
                str = "%s ---> %s"%(str,reverse_map["%d"%port])
        str = "%s ---> %s"%(str,reverse_map["%d"%p_node["port"]])
        print "Path: %s"%str
        rl_id =  "applied rules: "
        for (n,r,s) in p_node["hdr"].applied_rule_ids:
            rl_id = rl_id + " -> %s"%r
        print rl_id
        for i in range(len(p_node["hs_history"])):
            print "*** %d) AT PORT: %s\nHS: %s\n"%(i,reverse_map["%d"%p_node["visits"][i]],p_node["hs_history"][i])
        print "*** %d) AT PORT: %s\nHS: %s\n"%(i+1,reverse_map["%d"%p_node["port"]],p_node["hdr"])
        print "----------------------------------------------"

def loop_path_to_str(p_node, reverse_map):
    str = ""
    for port in p_node["visits"]:
        if str == "":
            str = reverse_map["%d"%port]
        else:
            str = "%s ---> %s"%(str,reverse_map["%d"%port])
    str = "%s ---> %s"%(str,reverse_map["%d"%p_node["port"]])
    return str

def trace_hs_back(applied_rule_list,hs,last_port):
    port = last_port
    hs_list = [hs]
    hs.applied_rule_ids = []
    for (tf,rule_id,in_port) in reversed(applied_rule_list):
        tmp = []
        for next_hs in hs_list:
            hp = tf.T_inv_rule(rule_id,next_hs,port)
            for (h,p_list) in hp:
                if in_port in p_list:
                    tmp.append(h)
        if tmp == []:
            break
        hs_list = tmp
        port = in_port
    return hs_list

def find_loop_original_header(NTF,TTF,propagation_node):
    applied_rule_ids = list(propagation_node["hdr"].applied_rule_ids)
    hs_list = trace_hs_back(applied_rule_ids,propagation_node["hdr"],propagation_node["port"])
    return hs_list

#def is_loop_infinite(NTF, TTF, propagation_node):
#    p_node = propagation_node
#    header_cpy = p_node["hdr"].copy()
#    while len(p_node['visits']) > 0:
#        cur_port = p_node['visits'].pop()
#
#         transfer back on the link
#        linked = TTF.T_inv(p_node["hdr"],p_node["port"])
#
#         transfer back on node
#        prev_hp = NTF.T_inv(linked[0][0],linked[0][1])
#
#        prev_hs = headerspace(NTF.length)
#        for (h,p) in prev_hp:
#            if p == cur_port:
#                prev_hs.add_hs(h)
#
#        p_node["hdr"] = prev_hs
#        p_node["port"] = cur_port

    #print "hret = %s and"%propagation_node["hdr"]
    #print "hret_inv = %s"%(p_node["hdr"])
#    if propagation_node["hdr"].is_subset_of(p_node["hdr"]):
#        return (True,p_node["hdr"])
#    else:
#        insect = propagation_node["hdr"].copy_intersect(p_node["hdr"])
#        if insect.count() == 0:
#            return (False,[])
#        else:
#            new_cpy = propagation_node.copy()
#            new_cpy["hdr"] = insect
#            return is_loop_infinite(NTF, TTF, new_cpy)

<|endoftext|>


<commit_before>from datetime import date, datetime
from math import floor

from django.core.management.base import BaseCommand
from django.conf import settings

from requests import HTTPError
from go_http.contacts import ContactsApiClient

from subscription.models import Subscription

SUBSCRIPTION_STANDARD = 1  # less than week 32 when reg
SUBSCRIPTION_LATER = 2  # 32-35 when reg

class Command(BaseCommand):
    help = "Ensure a mom's subscription is inline with protocol schedule"

    client_class = ContactsApiClient

    def year_from_month(self, month):
        if int(month) > 8:
            return 2014  # due in 2014
        else:
            return 2015

    def clean_day(self, day):
        if int(day) < 31:
            return int(day)
        else:
            self.stdout.write(
                "Contact has malformed due day data \
                    of %s so making it 14" % day)
            return 14

    def clean_month(self, month):
        return int(month)

    def calc_weeks(self, due_date, today=None):
        if today is None:
            today = date.today()
        # calc diff betwen now and due day
        diff = (due_date - today).days
        # get it in weeks
        diff_weeks = int(floor((diff / 7)))
        # get preg week
        preg_week = 40 - diff_weeks
        # You can't be less than two week preg
        if preg_week <= 1:
            return False
        elif preg_week > 41:
            return 41
        else:
            return preg_week

    def calc_sequence_start(self, weeks, schedule):
        # calculates which sms in the sequence to start with
        if schedule == SUBSCRIPTION_STANDARD:
            if weeks < 5:
                # Start from beginning
                seq_start = 1
            elif weeks < 41:
                # Message starts at week 5, sequence starts at 1
                START_OFFSET = 4
                PER_WEEK = 2
                seq_start = ((weeks - START_OFFSET) * PER_WEEK) - (PER_WEEK - 1)
            else:
                self.stdout.write("Fast forwarding to end\n")
                # Start of last week on schedule
                seq_start = 73
        elif schedule == SUBSCRIPTION_LATER:
            if weeks < 40:
                # Message starts at week 31, sequence starts at 1
                START_OFFSET = 30
                PER_WEEK = 3
                seq_start = ((weeks - START_OFFSET) * PER_WEEK) - (PER_WEEK - 1)
            else:
                self.stdout.write("Fast forwarding to end\n")
                # Start of last week on schedule
                seq_start = 28
        return seq_start

    def get_now(self):
        return datetime.now()

    def handle(self, *args, **options):
        subscribers = Subscription.objects.filter(
            active=True, completed=False, message_set__short_name='standard').all()

        # Make a reuseable contact api connection
        contacts = self.client_class(settings.VUMI_GO_API_TOKEN)
        counter = 0.0
        started = self.get_now()
        for subscriber in subscribers:
            self.stdout.write("Getting: %s\n" % (subscriber.contact_key,))
            try:
                contact = contacts.get_contact(subscriber.contact_key)
                if "extra" in contact \
                        and "due_date_day" in contact["extra"] \
                        and "due_date_month" in contact["extra"]:
                    year = self.year_from_month(
                        contact["extra"]["due_date_month"])
                    month = self.clean_month(
                        contact["extra"]["due_date_month"])
                    day = self.clean_day(contact["extra"]["due_date_day"])
                    due_date = date(year, month, day)
                    weeks = self.calc_weeks(due_date)
                    self.stdout.write("Mother due %s\n" % due_date.isoformat())
                    self.stdout.write("Week of preg %s\n" % weeks)
                elif "extra" in contact \
                        and "due_date_month" in contact["extra"]:
                    year = self.year_from_month(
                        contact["extra"]["due_date_month"])
                    month = self.clean_month(
                        contact["extra"]["due_date_month"])
                    day = 14
                    self.stdout.write(
                        "Contact %s has no due day data so making it 14" %
                        subscriber.contact_key)
                    weeks = self.calc_weeks(due_date)
                    self.stdout.write(
                        "Mother due %s" % due_date.isoformat())
                    self.stdout.write("Week of preg %s" % weeks)
                sub_type = int(contact["extra"]["subscription_type"])
                self.stdout.write("Sub type is %s\n" % sub_type)
                new_seq_num = self.calc_sequence_start(weeks, sub_type)
                self.stdout.write("Setting to seq %s from %s\n" % (new_seq_num, str(subscriber.next_sequence_number)))
                subscriber.next_sequence_number = new_seq_num
                subscriber.save()
                counter += 1.0
                delta = self.get_now() - started
                # Make sure we're not dividing by zero
                if delta.seconds > 0:
                    per_second = (counter / float((self.get_now() - started).seconds))
                else:
                    per_second = 'unknown'
                self.stdout.write("Updated %s subscribers at %s per second\n" % (counter, per_second))

            except HTTPError as err:
                self.stdout.write(
                    "Contact %s threw %s\n" % (subscriber.contact_key,
                                             err.response.status_code))
        self.stdout.write("Completed\n")
<commit_msg>Updated filters for set_seq<commit_after>from datetime import date, datetime
from math import floor

from django.core.management.base import BaseCommand
from django.conf import settings
from django.db.models import Q

from requests import HTTPError
from go_http.contacts import ContactsApiClient

from subscription.models import Subscription

SUBSCRIPTION_STANDARD = 1  # less than week 32 when reg
SUBSCRIPTION_LATER = 2  # 32-35 when reg

class Command(BaseCommand):
    help = "Ensure a mom's subscription is inline with protocol schedule"

    client_class = ContactsApiClient

    def year_from_month(self, month):
        if int(month) > 8:
            return 2014  # due in 2014
        else:
            return 2015

    def clean_day(self, day):
        if int(day) < 31:
            return int(day)
        else:
            self.stdout.write(
                "Contact has malformed due day data \
                    of %s so making it 14" % day)
            return 14

    def clean_month(self, month):
        return int(month)

    def calc_weeks(self, due_date, today=None):
        if today is None:
            today = date.today()
        # calc diff betwen now and due day
        diff = (due_date - today).days
        # get it in weeks
        diff_weeks = int(floor((diff / 7)))
        # get preg week
        preg_week = 40 - diff_weeks
        # You can't be less than two week preg
        if preg_week <= 1:
            return False
        elif preg_week > 41:
            return 41
        else:
            return preg_week

    def calc_sequence_start(self, weeks, schedule):
        # calculates which sms in the sequence to start with
        if schedule == SUBSCRIPTION_STANDARD:
            if weeks < 5:
                # Start from beginning
                seq_start = 1
            elif weeks < 41:
                # Message starts at week 5, sequence starts at 1
                START_OFFSET = 4
                PER_WEEK = 2
                seq_start = (
                    (weeks - START_OFFSET) * PER_WEEK) - (PER_WEEK - 1)
            else:
                self.stdout.write("Fast forwarding to end\n")
                # Start of last week on schedule
                seq_start = 73
        elif schedule == SUBSCRIPTION_LATER:
            if weeks < 40:
                # Message starts at week 31, sequence starts at 1
                START_OFFSET = 30
                PER_WEEK = 3
                seq_start = (
                    (weeks - START_OFFSET) * PER_WEEK) - (PER_WEEK - 1)
            else:
                self.stdout.write("Fast forwarding to end\n")
                # Start of last week on schedule
                seq_start = 28
        else:
            self.stdout.write("Unexpected schedule\n")
            seq_start = 0
        return seq_start

    def get_now(self):
        return datetime.now()

    def handle(self, *args, **options):
        subscribers = Subscription.objects.filter(
            Q(active=True), Q(completed=False),
            Q(message_set__short_name='standard') | Q(message_set__short_name='later'))

        # Make a reuseable contact api connection
        contacts = self.client_class(settings.VUMI_GO_API_TOKEN)
        counter = 0.0
        started = self.get_now()
        for subscriber in subscribers:
            self.stdout.write("Getting: %s\n" % (subscriber.contact_key,))
            try:
                contact = contacts.get_contact(subscriber.contact_key)
                if "extra" in contact \
                        and "due_date_day" in contact["extra"] \
                        and "due_date_month" in contact["extra"]:
                    year = self.year_from_month(
                        contact["extra"]["due_date_month"])
                    month = self.clean_month(
                        contact["extra"]["due_date_month"])
                    day = self.clean_day(contact["extra"]["due_date_day"])
                    due_date = date(year, month, day)
                    weeks = self.calc_weeks(due_date)
                    self.stdout.write("Mother due %s\n" % due_date.isoformat())
                    self.stdout.write("Week of preg %s\n" % weeks)
                elif "extra" in contact \
                        and "due_date_month" in contact["extra"]:
                    year = self.year_from_month(
                        contact["extra"]["due_date_month"])
                    month = self.clean_month(
                        contact["extra"]["due_date_month"])
                    day = 14
                    self.stdout.write(
                        "Contact %s has no due day data so making it 14" %
                        subscriber.contact_key)
                    weeks = self.calc_weeks(due_date)
                    self.stdout.write(
                        "Mother due %s" % due_date.isoformat())
                    self.stdout.write("Week of preg %s" % weeks)
                sub_type = int(contact["extra"]["subscription_type"])
                if sub_type is not int(subscriber.message_set_id):
                    self.stdout.write("Sub type %s does not match contact sub type %s\n" % (
                        sub_type, int(subscriber.message_set_id)))
                else:
                    self.stdout.write("Sub type is %s\n" % sub_type)
                    new_seq_num = self.calc_sequence_start(weeks, sub_type)
                    self.stdout.write("Setting to seq %s from %s\n" % (
                        new_seq_num, str(subscriber.next_sequence_number)))
                    subscriber.next_sequence_number = new_seq_num
                    subscriber.save()
                    counter += 1.0
                    delta = self.get_now() - started
                    # Make sure we're not dividing by zero
                    if delta.seconds > 0:
                        per_second = (
                            counter / float((self.get_now() - started).seconds))
                    else:
                        per_second = 'unknown'
                    self.stdout.write(
                        "Updated %s subscribers at %s per second\n" % (counter, per_second))

            except HTTPError as err:
                self.stdout.write(
                    "Contact %s threw %s\n" % (subscriber.contact_key,
                                               err.response.status_code))
        self.stdout.write("Completed\n")
<|endoftext|>


<commit_before># Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""# tf.experimental.numpy: NumPy API on TensorFlow.

This module provides a subset of NumPy API, built on top of TensorFlow
operations. APIs are based on and have been tested with NumPy 1.16 version.

The set of supported APIs may be expanded over time. Also future releases may
change the baseline version of NumPy API being supported. A list of some
systematic differences with NumPy is listed later in the "Differences with
NumPy" section.

## Getting Started

Please also see [TensorFlow NumPy Guide](
https://www.tensorflow.org/guide/tf_numpy).

In the code snippets below, we will assume that `tf.experimental.numpy` is
imported as `tnp` and NumPy is imported as `np`

```python
print(tnp.ones([2,1]) + np.ones([1, 2]))
```

## Types

The module provides an `ndarray` class which wraps an immutable `tf.Tensor`.
Additional functions are provided which accept array-like objects. Here
array-like objects includes `ndarrays` as defined by this module, as well as
`tf.Tensor`, in addition to types accepted by NumPy.

A subset of NumPy dtypes are supported. Type promotion follows NumPy
semantics.

```python
print(tnp.ones([1, 2], dtype=tnp.int16) + tnp.ones([2, 1], dtype=tnp.uint8))
```

## Array Interface

The `ndarray` class implements the `__array__` interface. This should allow
these objects to be passed into contexts that expect a NumPy or array-like
object (e.g. matplotlib).

```python
np.sum(tnp.ones([1, 2]) + np.ones([2, 1]))
```

## TF Interoperability

The TF-NumPy API calls can be interleaved with TensorFlow calls
without incurring Tensor data copies. This is true even if the `ndarray` or
`tf.Tensor` is placed on a non-CPU device.

In general, the expected behavior should be on par with that of code involving
`tf.Tensor` and running stateless TensorFlow functions on them.

```python
tnp.sum(tnp.ones([1, 2]) + tf.ones([2, 1]))
```

Note that the `__array_priority__` is currently chosen to be lower than
`tf.Tensor`. Hence the `+` operator above returns a `tf.Tensor`.

Additional examples of interopability include:

*  using `with tf.GradientTape()` scope to compute gradients through the
  TF-NumPy API calls.
*  using `tf.distribution.Strategy` scope for distributed execution
*  using `tf.vectorized_map()` for speeding up code using auto-vectorization

## Device Support

Given that `ndarray` and functions wrap TensorFlow constructs, the code will
have GPU and TPU support on par with TensorFlow. Device placement can be
controlled by using `with tf.device` scopes. Note that these devices could
be local or remote.

```python
with tf.device("GPU:0"):
  x = tnp.ones([1, 2])
print(tf.convert_to_tensor(x).device)
```

## Graph and Eager Modes

Eager mode execution should typically match NumPy semantics of executing
op-by-op. However the same code can be executed in graph mode, by putting it
inside a `tf.function`. The function body can contain NumPy code, and the inputs
can be `ndarray` as well.

```python
@tf.function
def f(x, y):
  return tnp.sum(x + y)

f(tnp.ones([1, 2]), tf.ones([2, 1]))
```
Python control flow based on `ndarray` values will be translated by
[autograph](https://www.tensorflow.org/code/tensorflow/python/autograph/g3doc/reference/index.md)
into `tf.cond` and `tf.while_loop` constructs. The code can be XLA compiled
for further optimizations.

However, note that graph mode execution can change behavior of certain
operations since symbolic execution may not have information that is computed
during runtime. Some differences are:

*   Shapes can be incomplete or unknown in graph mode. This means that
    `ndarray.shape`, `ndarray.size` and `ndarray.ndim` can return `ndarray`
    objects instead of returning integer (or tuple of integer) values.
*   `__len__`, `__iter__` and `__index__` properties of `ndarray`
    may similarly not be supported in graph mode. Code using these
    may need to change to explicit shape operations or control flow
    constructs.
*   Also note the [autograph limitations](
https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/autograph/g3doc/reference/limitations.md).

## Mutation and Variables

`ndarrays` currently wrap immutable `tf.Tensor`. Hence mutation
operations like slice assigns are not supported. This may change in the future.
Note however that one can directly construct a `tf.Variable` and use that with
the TF-NumPy APIs.

```python
tf_var = tf.Variable(2.0)
tf_var.assign_add(tnp.square(tf_var))
```

## Differences with NumPy

Here is a non-exhaustive list of differences:

*   Not all dtypes are currently supported. e.g. `np.float96`, `np.float128`.
    `np.object`, `np.str`, `np.recarray` types are not supported.
*   `ndarray` storage is in C order only. Fortran order, views, `stride_tricks`
    are not supported.
*   Only a subset of functions and modules are supported. This set will be
    expanded over time. For supported functions, some arguments or argument
    values may not be supported. These differences are generally provided in the
    function comments. Full `ufunc` support is also not provided.
*   Buffer mutation is currently not supported. `ndarrays` wrap immutable
    tensors. This means that output buffer arguments (e..g `out` in ufuncs) are
    not supported
*   NumPy C API is not supported. NumPy's Cython and Swig integration are not
    supported.
"""
# TODO(wangpeng): Append `np_export`ed symbols to the comments above.

# pylint: disable=g-direct-tensorflow-import

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.ops.array_ops import newaxis
from tensorflow.python.ops.numpy_ops import np_random as random
from tensorflow.python.ops.numpy_ops import np_utils
# pylint: disable=wildcard-import
from tensorflow.python.ops.numpy_ops.np_array_ops import *
from tensorflow.python.ops.numpy_ops.np_arrays import ndarray
from tensorflow.python.ops.numpy_ops.np_dtypes import *
from tensorflow.python.ops.numpy_ops.np_math_ops import *
# pylint: enable=wildcard-import
from tensorflow.python.ops.numpy_ops.np_utils import finfo
from tensorflow.python.ops.numpy_ops.np_utils import promote_types
from tensorflow.python.ops.numpy_ops.np_utils import result_type

# pylint: disable=redefined-builtin,undefined-variable
@np_utils.np_doc("max", link=np_utils.AliasOf("maximum"))
def max(a, axis=None, keepdims=None):
  return amax(a, axis=axis, keepdims=keepdims)

@np_utils.np_doc("min", link=np_utils.AliasOf("minimum"))
def min(a, axis=None, keepdims=None):
  return amin(a, axis=axis, keepdims=keepdims)

@np_utils.np_doc("round", link=np_utils.AliasOf("around"))
def round(a, decimals=0):
  return around(a, decimals=decimals)
# pylint: enable=redefined-builtin,undefined-variable
<commit_msg>One more<commit_after># Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""# tf.experimental.numpy: NumPy API on TensorFlow.

This module provides a subset of NumPy API, built on top of TensorFlow
operations. APIs are based on and have been tested with NumPy 1.16 version.

The set of supported APIs may be expanded over time. Also future releases may
change the baseline version of NumPy API being supported. A list of some
systematic differences with NumPy is listed later in the "Differences with
NumPy" section.

## Getting Started

Please also see [TensorFlow NumPy Guide](
https://www.tensorflow.org/guide/tf_numpy).

In the code snippets below, we will assume that `tf.experimental.numpy` is
imported as `tnp` and NumPy is imported as `np`

```python
print(tnp.ones([2,1]) + np.ones([1, 2]))
```

## Types

The module provides an `ndarray` class which wraps an immutable `tf.Tensor`.
Additional functions are provided which accept array-like objects. Here
array-like objects include `ndarrays` as defined by this module, as well as
`tf.Tensor`, in addition to types accepted by NumPy.

A subset of NumPy dtypes are supported. Type promotion follows NumPy
semantics.

```python
print(tnp.ones([1, 2], dtype=tnp.int16) + tnp.ones([2, 1], dtype=tnp.uint8))
```

## Array Interface

The `ndarray` class implements the `__array__` interface. This should allow
these objects to be passed into contexts that expect a NumPy or array-like
object (e.g. matplotlib).

```python
np.sum(tnp.ones([1, 2]) + np.ones([2, 1]))
```

## TF Interoperability

The TF-NumPy API calls can be interleaved with TensorFlow calls
without incurring Tensor data copies. This is true even if the `ndarray` or
`tf.Tensor` is placed on a non-CPU device.

In general, the expected behavior should be on par with that of code involving
`tf.Tensor` and running stateless TensorFlow functions on them.

```python
tnp.sum(tnp.ones([1, 2]) + tf.ones([2, 1]))
```

Note that the `__array_priority__` is currently chosen to be lower than
`tf.Tensor`. Hence the `+` operator above returns a `tf.Tensor`.

Additional examples of interoperability include:

*  using `with tf.GradientTape()` scope to compute gradients through the
  TF-NumPy API calls.
*  using `tf.distribution.Strategy` scope for distributed execution
*  using `tf.vectorized_map()` for speeding up code using auto-vectorization

## Device Support

Given that `ndarray` and functions wrap TensorFlow constructs, the code will
have GPU and TPU support on par with TensorFlow. Device placement can be
controlled by using `with tf.device` scopes. Note that these devices could
be local or remote.

```python
with tf.device("GPU:0"):
  x = tnp.ones([1, 2])
print(tf.convert_to_tensor(x).device)
```

## Graph and Eager Modes

Eager mode execution should typically match NumPy semantics of executing
op-by-op. However the same code can be executed in graph mode, by putting it
inside a `tf.function`. The function body can contain NumPy code, and the inputs
can be `ndarray` as well.

```python
@tf.function
def f(x, y):
  return tnp.sum(x + y)

f(tnp.ones([1, 2]), tf.ones([2, 1]))
```
Python control flow based on `ndarray` values will be translated by
[autograph](https://www.tensorflow.org/code/tensorflow/python/autograph/g3doc/reference/index.md)
into `tf.cond` and `tf.while_loop` constructs. The code can be XLA compiled
for further optimizations.

However, note that graph mode execution can change behavior of certain
operations since symbolic execution may not have information that is computed
during runtime. Some differences are:

*   Shapes can be incomplete or unknown in graph mode. This means that
    `ndarray.shape`, `ndarray.size` and `ndarray.ndim` can return `ndarray`
    objects instead of returning integer (or tuple of integer) values.
*   `__len__`, `__iter__` and `__index__` properties of `ndarray`
    may similarly not be supported in graph mode. Code using these
    may need to change to explicit shape operations or control flow
    constructs.
*   Also note the [autograph limitations](
https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/autograph/g3doc/reference/limitations.md).

## Mutation and Variables

`ndarrays` currently wrap immutable `tf.Tensor`. Hence mutation
operations like slice assigns are not supported. This may change in the future.
Note however that one can directly construct a `tf.Variable` and use that with
the TF-NumPy APIs.

```python
tf_var = tf.Variable(2.0)
tf_var.assign_add(tnp.square(tf_var))
```

## Differences with NumPy

Here is a non-exhaustive list of differences:

*   Not all dtypes are currently supported. e.g. `np.float96`, `np.float128`.
    `np.object`, `np.str`, `np.recarray` types are not supported.
*   `ndarray` storage is in C order only. Fortran order, views, `stride_tricks`
    are not supported.
*   Only a subset of functions and modules are supported. This set will be
    expanded over time. For supported functions, some arguments or argument
    values may not be supported. These differences are generally provided in the
    function comments. Full `ufunc` support is also not provided.
*   Buffer mutation is currently not supported. `ndarrays` wrap immutable
    tensors. This means that output buffer arguments (e.g. `out` in ufuncs) are
    not supported.
*   NumPy C API is not supported. NumPy's Cython and Swig integration are not
    supported.
"""
# TODO(wangpeng): Append `np_export`ed symbols to the comments above.

# pylint: disable=g-direct-tensorflow-import

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.ops.array_ops import newaxis
from tensorflow.python.ops.numpy_ops import np_random as random
from tensorflow.python.ops.numpy_ops import np_utils
# pylint: disable=wildcard-import
from tensorflow.python.ops.numpy_ops.np_array_ops import *
from tensorflow.python.ops.numpy_ops.np_arrays import ndarray
from tensorflow.python.ops.numpy_ops.np_dtypes import *
from tensorflow.python.ops.numpy_ops.np_math_ops import *
# pylint: enable=wildcard-import
from tensorflow.python.ops.numpy_ops.np_utils import finfo
from tensorflow.python.ops.numpy_ops.np_utils import promote_types
from tensorflow.python.ops.numpy_ops.np_utils import result_type

# pylint: disable=redefined-builtin,undefined-variable
@np_utils.np_doc("max", link=np_utils.AliasOf("maximum"))
def max(a, axis=None, keepdims=None):
  return amax(a, axis=axis, keepdims=keepdims)

@np_utils.np_doc("min", link=np_utils.AliasOf("minimum"))
def min(a, axis=None, keepdims=None):
  return amin(a, axis=axis, keepdims=keepdims)

@np_utils.np_doc("round", link=np_utils.AliasOf("around"))
def round(a, decimals=0):
  return around(a, decimals=decimals)
# pylint: enable=redefined-builtin,undefined-variable
<|endoftext|>


<commit_before>import logging
import os

from flask import Flask
from spreads.plugin import (HookPlugin, SubcommandHookMixin, PluginOption,
                            get_devices)
from spreads.vendor.pathlib import Path
from spreads.util import add_argument_from_option

app = Flask('spreadsplug.web', static_url_path='', static_folder='./client',
            template_folder='./client')
import web
import persistence
from worker import ProcessingWorker

logger = logging.getLogger('spreadsplug.web')

try:
    # Most reliable way to get IP address, requires third-party package with
    # C extension
    import netifaces

    def get_ip_address():
        try:
            iface = next(dev for dev in netifaces.interfaces()
                         if 'wlan' in dev or 'eth' in dev
                         or dev.startswith('br'))
        except StopIteration:
            return None
        return netifaces.ifaddresses(iface)[netifaces.AF_INET][0]['addr']
except ImportError:
    # Solution with built-ins, not as reliable
    import socket

    def get_ip_address():
        try:
            return next(
                ip for ip in socket.gethostbyname_ex(socket.gethostname())[2]
                if not ip.startswith("127."))
        except StopIteration:
            return None

class WebCommands(HookPlugin, SubcommandHookMixin):
    @classmethod
    def add_command_parser(cls, rootparser):
        cmdparser = rootparser.add_parser(
            'web', help="Start the web interface")
        cmdparser.set_defaults(subcommand=run_server)
        for key, option in cls.configuration_template().iteritems():
            try:
                add_argument_from_option('web', key, option, cmdparser)
            except:
                continue

    @classmethod
    def configuration_template(cls):
        return {
            'mode': PluginOption(
                value=["full", "scanner", "processor"],
                docstring="Mode to run server in",
                selectable=True),
            'debug': PluginOption(
                value=False,
                docstring="Run server in debugging mode",
                selectable=False),
            'project_dir': PluginOption(
                value=u"~/scans",
                docstring="Directory for project folders",
                selectable=False),
            'database': PluginOption(
                value=u"~/.config/spreads/workflows.db",
                docstring="Path to application database file",
                selectable=False),
            'postprocessing_server': PluginOption(
                value=u"",  # Cannot be None because of type deduction in
                            # option parser
                docstring="Address of the postprocessing server",
                selectable=False),
            'standalone_device': PluginOption(
                value=False,
                docstring="Server runs on a standalone device dedicated to "
                          "scanning (e.g. 'spreadpi').",
                selectable=False)
        }

def setup_app(config):
    # Set rootlogger to INFO
    if config['loglevel'].get() not in ('debug', 'info'):
        for handler in logging.getLogger().handlers:
            handler.setLevel(logging.INFO)

    mode = config['web']['mode'].get()
    logger.debug("Starting scanning station server in \"{0}\" mode"
                 .format(mode))
    db_path = Path(config['web']['database'].get()).expanduser()
    project_dir = os.path.expanduser(config['web']['project_dir'].get())
    if not os.path.exists(project_dir):
        os.mkdir(project_dir)

    app.config['DEBUG'] = config['web']['debug'].get()
    app.config['mode'] = mode
    app.config['database'] = db_path
    app.config['base_path'] = project_dir
    app.config['default_config'] = config
    app.config['standalone'] = config['web']['standalone_device'].get()

    if mode == 'scanner':
        app.config['postproc_server'] = (
            config['web']['postprocessing_server'].get())

def run_server(config):
    setup_app(config)
    if app.config['mode'] != 'scanner':
        worker = ProcessingWorker()
        worker.start()

    ip_address = get_ip_address()
    if (app.config['standalone'] and ip_address
            and config['driver'].get() in ['chdkcamera', 'a2200']):
        # Display the address of the web interface on the camera displays
        try:
            for cam in get_devices(config):
                cam.show_textbox(
                    "\n    You can now access the web interface at:"
                    "\n\n\n         http://{0}:5000".format(ip_address))
        except:
            logger.warn("No devices could be found at startup.")

    try:
        import waitress
        waitress.serve(app, port=5000)
    finally:
        if app.config['mode'] != 'scanner':
            worker.stop()
        if app.config['DEBUG']:
            logger.info("Waiting for remaining connections to close...")
<commit_msg>web: Compress non-image/zip responses<commit_after>import gzip
import logging
import os
from cStringIO import StringIO as IO

from flask import Flask, request
from spreads.plugin import (HookPlugin, SubcommandHookMixin, PluginOption,
                            get_devices)
from spreads.vendor.pathlib import Path
from spreads.util import add_argument_from_option

app = Flask('spreadsplug.web', static_url_path='', static_folder='./client',
            template_folder='./client')
import web
import persistence
from worker import ProcessingWorker

logger = logging.getLogger('spreadsplug.web')

try:
    # Most reliable way to get IP address, requires third-party package with
    # C extension
    import netifaces

    def get_ip_address():
        try:
            iface = next(dev for dev in netifaces.interfaces()
                         if 'wlan' in dev or 'eth' in dev
                         or dev.startswith('br'))
        except StopIteration:
            return None
        return netifaces.ifaddresses(iface)[netifaces.AF_INET][0]['addr']
except ImportError:
    # Solution with built-ins, not as reliable
    import socket

    def get_ip_address():
        try:
            return next(
                ip for ip in socket.gethostbyname_ex(socket.gethostname())[2]
                if not ip.startswith("127."))
        except StopIteration:
            return None

class WebCommands(HookPlugin, SubcommandHookMixin):
    @classmethod
    def add_command_parser(cls, rootparser):
        cmdparser = rootparser.add_parser(
            'web', help="Start the web interface")
        cmdparser.set_defaults(subcommand=run_server)
        for key, option in cls.configuration_template().iteritems():
            try:
                add_argument_from_option('web', key, option, cmdparser)
            except:
                continue

    @classmethod
    def configuration_template(cls):
        return {
            'mode': PluginOption(
                value=["full", "scanner", "processor"],
                docstring="Mode to run server in",
                selectable=True),
            'debug': PluginOption(
                value=False,
                docstring="Run server in debugging mode",
                selectable=False),
            'project_dir': PluginOption(
                value=u"~/scans",
                docstring="Directory for project folders",
                selectable=False),
            'database': PluginOption(
                value=u"~/.config/spreads/workflows.db",
                docstring="Path to application database file",
                selectable=False),
            'postprocessing_server': PluginOption(
                value=u"",  # Cannot be None because of type deduction in
                            # option parser
                docstring="Address of the postprocessing server",
                selectable=False),
            'standalone_device': PluginOption(
                value=False,
                docstring="Server runs on a standalone device dedicated to "
                          "scanning (e.g. 'spreadpi').",
                selectable=False)
        }

@app.after_request
def gzip_response(response):
    accept_encoding = request.headers.get('Accept-Encoding', '')

    if 'gzip' not in accept_encoding.lower():
        return response

    response.direct_passthrough = False
    skip_compress = (response.status_code < 200 or
                     response.status_code >= 300 or
                     response.mimetype.startswith("image/") or
                     response.mimetype == "application/zip" or
                     'Content-Encoding' in response.headers)
    if skip_compress:
        return response
    gzip_buffer = IO()
    gzip_file = gzip.GzipFile(mode='wb',
                              fileobj=gzip_buffer)
    gzip_file.write(response.data)
    gzip_file.close()

    response.data = gzip_buffer.getvalue()
    response.headers['Content-Encoding'] = 'gzip'
    response.headers['Vary'] = 'Accept-Encoding'
    response.headers['Content-Length'] = len(response.data)

    return response

def setup_app(config):
    # Set rootlogger to INFO
    if config['loglevel'].get() not in ('debug', 'info'):
        for handler in logging.getLogger().handlers:
            handler.setLevel(logging.INFO)

    mode = config['web']['mode'].get()
    logger.debug("Starting scanning station server in \"{0}\" mode"
                 .format(mode))
    db_path = Path(config['web']['database'].get()).expanduser()
    project_dir = os.path.expanduser(config['web']['project_dir'].get())
    if not os.path.exists(project_dir):
        os.mkdir(project_dir)

    app.config['DEBUG'] = config['web']['debug'].get()
    app.config['mode'] = mode
    app.config['database'] = db_path
    app.config['base_path'] = project_dir
    app.config['default_config'] = config
    app.config['standalone'] = config['web']['standalone_device'].get()

    if mode == 'scanner':
        app.config['postproc_server'] = (
            config['web']['postprocessing_server'].get())

def run_server(config):
    setup_app(config)
    if app.config['mode'] != 'scanner':
        worker = ProcessingWorker()
        worker.start()

    ip_address = get_ip_address()
    if (app.config['standalone'] and ip_address
            and config['driver'].get() in ['chdkcamera', 'a2200']):
        # Display the address of the web interface on the camera displays
        try:
            for cam in get_devices(config):
                cam.show_textbox(
                    "\n    You can now access the web interface at:"
                    "\n\n\n         http://{0}:5000".format(ip_address))
        except:
            logger.warn("No devices could be found at startup.")

    try:
        import waitress
        waitress.serve(app, port=5000)
    finally:
        if app.config['mode'] != 'scanner':
            worker.stop()
        if app.config['DEBUG']:
            logger.info("Waiting for remaining connections to close...")
<|endoftext|>