<commit_before>import pytest

from setuptools_scm.utils import do_ex
from setuptools_scm.utils import has_command

python_hg, err, ret = do_ex("hg debuginstall --template {pythonexe}")

if ret:
    skip_no_hggit = True
else:
    out, err, ret = do_ex([python_hg.strip(), "-c", "import hggit"])
    print(out, err, ret)
    skip_no_hggit = bool(ret)

@pytest.mark.skipif(not has_command("hg", warn=False), reason="hg executable not found")
@pytest.mark.skipif(skip_no_hggit, reason="hg-git not installed")
def test_base(repositories_hg_git):
    wd, wd_git = repositories_hg_git

    assert wd_git.version == "0.1.dev0"
    assert wd.version == "0.1.dev0"

    wd_git.commit_testfile()
    wd("hg pull -u")

    version_git = wd_git.version
    version = wd.version

    assert version_git.startswith("0.1.dev1+g")
    assert version.startswith("0.1.dev1+g")

    assert not version_git.endswith("1-")
    assert not version.endswith("1-")

    wd_git("git tag v0.1")
    wd("hg pull -u")
    assert wd_git.version == "0.1"
    assert wd.version == "0.1"

    wd_git.write("test.txt", "test2")
    wd.write("test.txt", "test2")
    assert wd_git.version.startswith("0.2.dev0+g")
    assert wd.version.startswith("0.2.dev0+g")

    wd_git.commit_testfile()
    wd("hg pull")
    wd("hg up -C")
    assert wd_git.version.startswith("0.2.dev1+g")
    assert wd.version.startswith("0.2.dev1+g")

    wd_git("git tag version-0.2")
    wd("hg pull -u")
    assert wd_git.version.startswith("0.2")
    assert wd.version.startswith("0.2")

    wd_git.commit_testfile()
    wd_git("git tag version-0.2.post210+gbe48adfpost3+g0cc25f2")
    wd("hg pull -u")
    with pytest.warns(
        UserWarning, match="tag '.*' will be stripped of its suffix '.*'"
    ):
        assert wd_git.version.startswith("0.2")

    with pytest.warns(
        UserWarning, match="tag '.*' will be stripped of its suffix '.*'"
    ):
        assert wd.version.startswith("0.2")

    wd_git.commit_testfile()
    wd_git("git tag 17.33.0-rc")
    wd("hg pull -u")
    assert wd_git.version == "17.33.0rc0"
    assert wd.version == "17.33.0rc0"
<commit_msg>enhance mercurial detection for hg_git tests<commit_after>import pytest

from setuptools_scm.utils import do_ex
from setuptools_scm.utils import has_command

@pytest.fixture(scope="module", autouse=True)
def _check_hg_git():
    if not has_command("hg", warn=False):
        pytest.skip("hg executable not found")

    python_hg, err, ret = do_ex("hg debuginstall --template {pythonexe}")

    if ret:
        skip_no_hggit = True
    else:
        out, err, ret = do_ex([python_hg.strip(), "-c", "import hggit"])
    skip_no_hggit = bool(ret)
    if skip_no_hggit:
        pytest.skip("hg-git not installed")

def test_base(repositories_hg_git):
    wd, wd_git = repositories_hg_git

    assert wd_git.version == "0.1.dev0"
    assert wd.version == "0.1.dev0"

    wd_git.commit_testfile()
    wd("hg pull -u")

    version_git = wd_git.version
    version = wd.version

    assert version_git.startswith("0.1.dev1+g")
    assert version.startswith("0.1.dev1+g")

    assert not version_git.endswith("1-")
    assert not version.endswith("1-")

    wd_git("git tag v0.1")
    wd("hg pull -u")
    assert wd_git.version == "0.1"
    assert wd.version == "0.1"

    wd_git.write("test.txt", "test2")
    wd.write("test.txt", "test2")
    assert wd_git.version.startswith("0.2.dev0+g")
    assert wd.version.startswith("0.2.dev0+g")

    wd_git.commit_testfile()
    wd("hg pull")
    wd("hg up -C")
    assert wd_git.version.startswith("0.2.dev1+g")
    assert wd.version.startswith("0.2.dev1+g")

    wd_git("git tag version-0.2")
    wd("hg pull -u")
    assert wd_git.version.startswith("0.2")
    assert wd.version.startswith("0.2")

    wd_git.commit_testfile()
    wd_git("git tag version-0.2.post210+gbe48adfpost3+g0cc25f2")
    wd("hg pull -u")
    with pytest.warns(
        UserWarning, match="tag '.*' will be stripped of its suffix '.*'"
    ):
        assert wd_git.version.startswith("0.2")

    with pytest.warns(
        UserWarning, match="tag '.*' will be stripped of its suffix '.*'"
    ):
        assert wd.version.startswith("0.2")

    wd_git.commit_testfile()
    wd_git("git tag 17.33.0-rc")
    wd("hg pull -u")
    assert wd_git.version == "17.33.0rc0"
    assert wd.version == "17.33.0rc0"
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import logging

from django.views.generic import TemplateView

from haystack.query import SearchQuerySet, EmptySearchQuerySet

from dynamiq.utils import get_advanced_search_formset_class, FormsetQBuilder, ParsedStringQBuilder

from .forms import MEPSearchForm, MEPSearchAdvancedFormset, MEPSimpleSearchForm
from .shortcuts import TopRated, WorstRated

log = logging.getLogger(__name__)

class SearchView(TemplateView):

    template_name = 'search/search.html'
    list_template_name = "blocks/representative_list.html"

    def get_template_names(self):
        """
        Dispatch template according to the kind of request: ajax or normal.
        """
        if self.request.is_ajax():
            return [self.list_template_name]
        else:
            return [self.template_name]

    def get_context_data(self, **kwargs):
        query = None
        sort = MEPSearchAdvancedFormset.options_form_class.SORT_INITIAL
        limit = MEPSearchAdvancedFormset.options_form_class.LIMIT_INITIAL
        label = ""

        formset_class = get_advanced_search_formset_class(self.request.user, MEPSearchAdvancedFormset, MEPSearchForm)
        if "q" in self.request.GET:
            form = MEPSimpleSearchForm(self.request.GET)
            if form.is_valid():
                F = ParsedStringQBuilder(form.cleaned_data['q'], MEPSearchForm)
                query, label = F()
                formset = formset_class()
                _limit = form.cleaned_data.get("limit")
                if _limit is not None: limit = _limit
                sort = form.cleaned_data.get("sort") or sort
        else:
            form = MEPSimpleSearchForm()
            formset = formset_class(self.request.GET or None)
            formset.full_clean()
            if formset.is_valid():
                F = FormsetQBuilder(formset)
                query, label = F()
                sort = formset.options_form.cleaned_data.get("sort", sort)
                limit = formset.options_form.cleaned_data.get("limit", limit)

        if query:
            results = SearchQuerySet().filter(query)
            if sort:
                results = results.order_by(sort)
            if not limit:
                # When iterating over SearchQuerySet, haystack will fetch
                # results 10 by 10. This fetchs them all in one call:
                results = results[:]
        else:
            results = EmptySearchQuerySet()
        return {
            "dynamiq": {
                "results": results,
                "label": label,
                "formset": formset,
                "form": form,
                "shortcuts": [
                    TopRated({"request": self.request}),
                    WorstRated({"request": self.request})
                ]
            },
            "list_template_name": self.list_template_name,
            "per_page": limit
        }

class XhrSearchView(SearchView):

    template_name = "search/xhr.html"
<commit_msg>[esthetic] less cumbersome code<commit_after># -*- coding: utf-8 -*-
import logging

from django.views.generic import TemplateView

from haystack.query import SearchQuerySet, EmptySearchQuerySet

from dynamiq.utils import get_advanced_search_formset_class, FormsetQBuilder, ParsedStringQBuilder

from .forms import MEPSearchForm, MEPSearchAdvancedFormset, MEPSimpleSearchForm
from .shortcuts import TopRated, WorstRated

log = logging.getLogger(__name__)

class SearchView(TemplateView):

    template_name = 'search/search.html'
    list_template_name = "blocks/representative_list.html"

    def get_template_names(self):
        """
        Dispatch template according to the kind of request: ajax or normal.
        """
        if self.request.is_ajax():
            return [self.list_template_name]
        else:
            return [self.template_name]

    def get_context_data(self, **kwargs):
        query = None
        sort = MEPSearchAdvancedFormset.options_form_class.SORT_INITIAL
        limit = MEPSearchAdvancedFormset.options_form_class.LIMIT_INITIAL
        label = ""

        formset_class = get_advanced_search_formset_class(self.request.user, MEPSearchAdvancedFormset, MEPSearchForm)
        if "q" in self.request.GET:
            form = MEPSimpleSearchForm(self.request.GET)
            if form.is_valid():
                F = ParsedStringQBuilder(form.cleaned_data['q'], MEPSearchForm)
                query, label = F()
                formset = formset_class()
                if "limit" in form.cleaned_data:
                    limit = form.cleaned_data["limit"]
                sort = form.cleaned_data.get("sort") or sort
        else:
            form = MEPSimpleSearchForm()
            formset = formset_class(self.request.GET or None)
            formset.full_clean()
            if formset.is_valid():
                F = FormsetQBuilder(formset)
                query, label = F()
                sort = formset.options_form.cleaned_data.get("sort", sort)
                limit = formset.options_form.cleaned_data.get("limit", limit)

        if query:
            results = SearchQuerySet().filter(query)
            if sort:
                results = results.order_by(sort)
            if not limit:
                # When iterating over SearchQuerySet, haystack will fetch
                # results 10 by 10. This fetchs them all in one call:
                results = results[:]
        else:
            results = EmptySearchQuerySet()
        return {
            "dynamiq": {
                "results": results,
                "label": label,
                "formset": formset,
                "form": form,
                "shortcuts": [
                    TopRated({"request": self.request}),
                    WorstRated({"request": self.request})
                ]
            },
            "list_template_name": self.list_template_name,
            "per_page": limit
        }

class XhrSearchView(SearchView):

    template_name = "search/xhr.html"
<|endoftext|>


<commit_before>#
# GENIPUBLIC-COPYRIGHT
# Copyright (c) 2008-2009 University of Utah and the Flux Group.
# All rights reserved.
#
# Permission to use, copy, modify and distribute this software is hereby
# granted provided that (1) source code retains these copyright, permission,
# and disclaimer notices, and (2) redistributions including binaries
# reproduce the notices in supporting documentation.
#
# THE UNIVERSITY OF UTAH ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
# CONDITION.  THE UNIVERSITY OF UTAH DISCLAIMS ANY LIABILITY OF ANY KIND
# FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.

from urlparse import urlsplit, urlunsplit
from urllib import splitport

# Debugging output.
debug           = 0
impotent        = 0

HOME            = os.environ["HOME"]
# Path to my certificate
CERTIFICATE     = HOME + "/.ssl/encrypted.pem"
# Got tired of typing this over and over so I stuck it in a file.
PASSPHRASEFILE  = HOME + "/.ssl/password"
passphrase      = ""

CONFIGFILE      = ".protogeni-config.py"
GLOBALCONF      = HOME + "/" + CONFIGFILE
LOCALCONF       = CONFIGFILE
EXTRACONF       = None
SLICENAME       = "mytestslice"

selfcredentialfile = None
slicecredentialfile = None

if "Usage" not in dir():
    def Usage():
        print "usage: " + sys.argv[ 0 ] + " [option...]"
        print """Options:
    -c file, --credentials=file         read self-credentials from file
                                            [default: query from SA]
    -d, --debug                         be verbose about XML methods invoked
    -f file, --certificate=file         read SSL certificate from file
                                            [default: ~/.ssl/encrypted.pem]
    -h, --help                          show options and usage"""
        if "ACCEPTSLICENAME" in globals():
            print """    -n name, --slicename=name           specify human-readable name of slice
                                            [default: mytestslice]"""
        print """    -p file, --passphrase=file          read passphrase from file
                                            [default: ~/.ssl/password]
    -r file, --read-commands=file       specify additional configuration file
    -s file, --slicecredentials=file    read slice credentials from file
                                            [default: query from SA]"""

try:
    opts, args = getopt.getopt( sys.argv[ 1: ], "c:df:hn:p:r:s:",
                                [ "credentials=", "debug", "certificate=",
                                  "help", "passphrase=", "read-commands=",
                                  "slicecredentials=", "slicename=" ] )
except getopt.GetoptError, err:
    print >> sys.stderr, str( err )
    Usage()
    sys.exit( 1 )

if "PROTOGENI_CERTIFICATE" in os.environ:
    CERTIFICATE = os.environ[ "PROTOGENI_CERTIFICATE" ]
if "PROTOGENI_PASSPHRASE" in os.environ:
    PASSPHRASEFILE = os.environ[ "PROTOGENI_PASSPHRASE" ]

for opt, arg in opts:
    if opt in ( "-c", "--credentials" ):
        selfcredentialfile = arg
    elif opt in ( "-d", "--debug" ):
        debug = 1
    elif opt in ( "-f", "--certificate" ):
        CERTIFICATE = arg
    elif opt in ( "-h", "--help" ):
        Usage()
        sys.exit( 0 )
    elif opt in ( "-n", "--slicename" ):
        SLICENAME = arg
    elif opt in ( "-p", "--passphrase" ):
        PASSPHRASEFILE = arg
    elif opt in ( "-r", "--read-commands" ):
        EXTRACONF = arg
    elif opt in ( "-s", "--slicecredentials" ):
        slicecredentialfile = arg

cert = X509.load_cert( CERTIFICATE )

# XMLRPC server: use www.emulab.net for the clearinghouse, and
# the issuer of the certificate we'll identify with for everything else
XMLRPC_SERVER   = { "ch" : "www.emulab.net",
                    "default" : cert.get_issuer().CN }
SERVER_PATH     = { "default" : ":443/protogeni/xmlrpc/" }

if os.path.exists( GLOBALCONF ):
    execfile( GLOBALCONF )
if os.path.exists( LOCALCONF ):
    execfile( LOCALCONF )
if EXTRACONF and os.path.exists( EXTRACONF ):
    execfile( EXTRACONF )

def Fatal(message):
    print >> sys.stderr, message
    sys.exit(1)

def PassPhraseCB(v, prompt1='Enter passphrase:', prompt2='Verify passphrase:'):
    passphrase = open(PASSPHRASEFILE).readline()
    passphrase = passphrase.strip()
    return passphrase

#
# Call the rpc server.
#
def do_method(module, method, params, URI=None, quiet=False):
    if debug:
        print module + " " + method + " " + str(params);
        pass

    if not os.path.exists(CERTIFICATE):
        return Fatal("error: missing emulab certificate: %s\n" % CERTIFICATE)

    from M2Crypto.m2xmlrpclib import SSL_Transport
    from M2Crypto import SSL

    if URI == None:
        if module in XMLRPC_SERVER:
            addr = XMLRPC_SERVER[ module ]
        else:
            addr = XMLRPC_SERVER[ "default" ]

        if module in SERVER_PATH:
            path = SERVER_PATH[ module ]
        else:
            path = SERVER_PATH[ "default" ]

        URI = "https://" + addr + path + module
    elif module:
        URI = URI + "/" + module
        pass

    scheme, netloc, path, query, fragment = urlsplit(URI)
    if scheme == "https":
        host,port = splitport(netloc)
        if not port:
            netloc = netloc + ":443"
            URI = urlunsplit((scheme, netloc, path, query, fragment));
            pass
        pass

    ctx = SSL.Context("sslv23")
    ctx.load_cert(CERTIFICATE, CERTIFICATE, PassPhraseCB)
    ctx.set_verify(SSL.verify_none, 16)
    ctx.set_allow_unknown_ca(0)

    # Get a handle on the server,
    server = xmlrpclib.ServerProxy(URI, SSL_Transport(ctx), verbose=0)

    # Get a pointer to the function we want to invoke.
    meth      = getattr(server, method)
    meth_args = [ params ]

    #
    # Make the call.
    #
    try:
        response = apply(meth, meth_args)
        pass
    except xmlrpclib.Fault, e:
        if not quiet: print >> sys.stderr, e.faultString
        return (-1, None)
    except xmlrpclib.ProtocolError, e:
        if not quiet: print >> sys.stderr, e.errmsg
        return (-1, None)

    #
    # Parse the Response, which is a Dictionary. See EmulabResponse in the
    # emulabclient.py module. The XML standard converts classes to a plain
    # Dictionary, hence the code below.
    #
    if response[ "code" ] and len(response["output"]):
        if not quiet: print >> sys.stderr, response["output"] + ":",
        pass

    rval = response["code"]

    #
    # If the code indicates failure, look for a "value". Use that as the
    # return value instead of the code.
    #
    if rval:
        if response["value"]:
            rval = response["value"]
            pass
        pass
    return (rval, response)

def get_self_credential():
    if selfcredentialfile:
        f = open( selfcredentialfile )
        c = f.read()
        f.close()
        return c
    params = {}
    params["uuid"] = "0b2eb97e-ed30-11db-96cb-001143e453fe"
    rval,response = do_method("sa", "GetCredential", params)
    if rval:
        Fatal("Could not get my credential")
        pass
    return response["value"]

def resolve_slice( name, selfcredential ):
    params = {}
    params["credential"] = mycredential
    params["type"]       = "Slice"
    params["hrn"]        = name
    rval,response = do_method("sa", "Resolve", params)
    if rval:
        Fatal("Slice does not exist");
        pass
    else:
        return response["value"]

def get_slice_credential( slice, selfcredential ):
    if slicecredentialfile:
        f = open( slicecredentialfile )
        c = f.read()
        f.close()
        return c

    params = {}
    params["credential"] = selfcredential
    params["type"]       = "Slice"
    params["uuid"]       = slice["uuid"]
    rval,response = do_method("sa", "GetCredential", params)
    if rval:
        Fatal("Could not get Slice credential")
        pass
    return response["value"]
<commit_msg>Fix bug where debug mode would fail for do_method with URI but no module.<commit_after>#
# GENIPUBLIC-COPYRIGHT
# Copyright (c) 2008-2009 University of Utah and the Flux Group.
# All rights reserved.
#
# Permission to use, copy, modify and distribute this software is hereby
# granted provided that (1) source code retains these copyright, permission,
# and disclaimer notices, and (2) redistributions including binaries
# reproduce the notices in supporting documentation.
#
# THE UNIVERSITY OF UTAH ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
# CONDITION.  THE UNIVERSITY OF UTAH DISCLAIMS ANY LIABILITY OF ANY KIND
# FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.

from urlparse import urlsplit, urlunsplit
from urllib import splitport

# Debugging output.
debug           = 0
impotent        = 0

HOME            = os.environ["HOME"]
# Path to my certificate
CERTIFICATE     = HOME + "/.ssl/encrypted.pem"
# Got tired of typing this over and over so I stuck it in a file.
PASSPHRASEFILE  = HOME + "/.ssl/password"
passphrase      = ""

CONFIGFILE      = ".protogeni-config.py"
GLOBALCONF      = HOME + "/" + CONFIGFILE
LOCALCONF       = CONFIGFILE
EXTRACONF       = None
SLICENAME       = "mytestslice"

selfcredentialfile = None
slicecredentialfile = None

if "Usage" not in dir():
    def Usage():
        print "usage: " + sys.argv[ 0 ] + " [option...]"
        print """Options:
    -c file, --credentials=file         read self-credentials from file
                                            [default: query from SA]
    -d, --debug                         be verbose about XML methods invoked
    -f file, --certificate=file         read SSL certificate from file
                                            [default: ~/.ssl/encrypted.pem]
    -h, --help                          show options and usage"""
        if "ACCEPTSLICENAME" in globals():
            print """    -n name, --slicename=name           specify human-readable name of slice
                                            [default: mytestslice]"""
        print """    -p file, --passphrase=file          read passphrase from file
                                            [default: ~/.ssl/password]
    -r file, --read-commands=file       specify additional configuration file
    -s file, --slicecredentials=file    read slice credentials from file
                                            [default: query from SA]"""

try:
    opts, args = getopt.getopt( sys.argv[ 1: ], "c:df:hn:p:r:s:",
                                [ "credentials=", "debug", "certificate=",
                                  "help", "passphrase=", "read-commands=",
                                  "slicecredentials=", "slicename=" ] )
except getopt.GetoptError, err:
    print >> sys.stderr, str( err )
    Usage()
    sys.exit( 1 )

if "PROTOGENI_CERTIFICATE" in os.environ:
    CERTIFICATE = os.environ[ "PROTOGENI_CERTIFICATE" ]
if "PROTOGENI_PASSPHRASE" in os.environ:
    PASSPHRASEFILE = os.environ[ "PROTOGENI_PASSPHRASE" ]

for opt, arg in opts:
    if opt in ( "-c", "--credentials" ):
        selfcredentialfile = arg
    elif opt in ( "-d", "--debug" ):
        debug = 1
    elif opt in ( "-f", "--certificate" ):
        CERTIFICATE = arg
    elif opt in ( "-h", "--help" ):
        Usage()
        sys.exit( 0 )
    elif opt in ( "-n", "--slicename" ):
        SLICENAME = arg
    elif opt in ( "-p", "--passphrase" ):
        PASSPHRASEFILE = arg
    elif opt in ( "-r", "--read-commands" ):
        EXTRACONF = arg
    elif opt in ( "-s", "--slicecredentials" ):
        slicecredentialfile = arg

cert = X509.load_cert( CERTIFICATE )

# XMLRPC server: use www.emulab.net for the clearinghouse, and
# the issuer of the certificate we'll identify with for everything else
XMLRPC_SERVER   = { "ch" : "www.emulab.net",
                    "default" : cert.get_issuer().CN }
SERVER_PATH     = { "default" : ":443/protogeni/xmlrpc/" }

if os.path.exists( GLOBALCONF ):
    execfile( GLOBALCONF )
if os.path.exists( LOCALCONF ):
    execfile( LOCALCONF )
if EXTRACONF and os.path.exists( EXTRACONF ):
    execfile( EXTRACONF )

def Fatal(message):
    print >> sys.stderr, message
    sys.exit(1)

def PassPhraseCB(v, prompt1='Enter passphrase:', prompt2='Verify passphrase:'):
    passphrase = open(PASSPHRASEFILE).readline()
    passphrase = passphrase.strip()
    return passphrase

#
# Call the rpc server.
#
def do_method(module, method, params, URI=None, quiet=False):
    if debug:
        if module:
            print module + " " + method + " " + str(params);
        else:
            print URI + " " + method + " " + str(params);
        pass

    if not os.path.exists(CERTIFICATE):
        return Fatal("error: missing emulab certificate: %s\n" % CERTIFICATE)

    from M2Crypto.m2xmlrpclib import SSL_Transport
    from M2Crypto import SSL

    if URI == None:
        if module in XMLRPC_SERVER:
            addr = XMLRPC_SERVER[ module ]
        else:
            addr = XMLRPC_SERVER[ "default" ]

        if module in SERVER_PATH:
            path = SERVER_PATH[ module ]
        else:
            path = SERVER_PATH[ "default" ]

        URI = "https://" + addr + path + module
    elif module:
        URI = URI + "/" + module
        pass

    scheme, netloc, path, query, fragment = urlsplit(URI)
    if scheme == "https":
        host,port = splitport(netloc)
        if not port:
            netloc = netloc + ":443"
            URI = urlunsplit((scheme, netloc, path, query, fragment));
            pass
        pass

    ctx = SSL.Context("sslv23")
    ctx.load_cert(CERTIFICATE, CERTIFICATE, PassPhraseCB)
    ctx.set_verify(SSL.verify_none, 16)
    ctx.set_allow_unknown_ca(0)

    # Get a handle on the server,
    server = xmlrpclib.ServerProxy(URI, SSL_Transport(ctx), verbose=0)

    # Get a pointer to the function we want to invoke.
    meth      = getattr(server, method)
    meth_args = [ params ]

    #
    # Make the call.
    #
    try:
        response = apply(meth, meth_args)
        pass
    except xmlrpclib.Fault, e:
        if not quiet: print >> sys.stderr, e.faultString
        return (-1, None)
    except xmlrpclib.ProtocolError, e:
        if not quiet: print >> sys.stderr, e.errmsg
        return (-1, None)

    #
    # Parse the Response, which is a Dictionary. See EmulabResponse in the
    # emulabclient.py module. The XML standard converts classes to a plain
    # Dictionary, hence the code below.
    #
    if response[ "code" ] and len(response["output"]):
        if not quiet: print >> sys.stderr, response["output"] + ":",
        pass

    rval = response["code"]

    #
    # If the code indicates failure, look for a "value". Use that as the
    # return value instead of the code.
    #
    if rval:
        if response["value"]:
            rval = response["value"]
            pass
        pass
    return (rval, response)

def get_self_credential():
    if selfcredentialfile:
        f = open( selfcredentialfile )
        c = f.read()
        f.close()
        return c
    params = {}
    params["uuid"] = "0b2eb97e-ed30-11db-96cb-001143e453fe"
    rval,response = do_method("sa", "GetCredential", params)
    if rval:
        Fatal("Could not get my credential")
        pass
    return response["value"]

def resolve_slice( name, selfcredential ):
    params = {}
    params["credential"] = mycredential
    params["type"]       = "Slice"
    params["hrn"]        = name
    rval,response = do_method("sa", "Resolve", params)
    if rval:
        Fatal("Slice does not exist");
        pass
    else:
        return response["value"]

def get_slice_credential( slice, selfcredential ):
    if slicecredentialfile:
        f = open( slicecredentialfile )
        c = f.read()
        f.close()
        return c

    params = {}
    params["credential"] = selfcredential
    params["type"]       = "Slice"
    params["uuid"]       = slice["uuid"]
    rval,response = do_method("sa", "GetCredential", params)
    if rval:
        Fatal("Could not get Slice credential")
        pass
    return response["value"]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""Tests for Models
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from collections import namedtuple

from seq2seq import losses as seq2seq_losses
from seq2seq.data import data_utils, vocab
from seq2seq.training import utils as training_utils
from seq2seq.test import utils as test_utils
from seq2seq.models import BasicSeq2Seq, AttentionSeq2Seq
from seq2seq.decoders import FixedDecoderInputs, DynamicDecoderInputs

import tensorflow as tf
import numpy as np

class EncoderDecoderTests(tf.test.TestCase):
  """Base class for EncoderDecoder tests. Tests for specific classes should
  inherit from this and tf.test.TestCase.
  """

  def setUp(self):
    super(EncoderDecoderTests, self).setUp()
    tf.logging.set_verbosity(tf.logging.INFO)
    self.batch_size = 4
    self.input_depth = 32
    self.max_decode_length = 40

    # Create vocabulary
    self.vocab_list = [str(_) for _ in range(10)]
    self.vocab_list += ["笑う", "泣く", "了解", "はい", "＾＿＾"]
    self.vocab_size = len(self.vocab_list)
    self.vocab_file = test_utils.create_temporary_vocab_file(self.vocab_list)
    self.vocab_info = vocab.get_vocab_info(self.vocab_file.name)

  def tearDown(self):
    self.vocab_file.close()

  def create_model(self, _params=None):
    """Creates model class to be tested. Subclasses must implement this method.
    """
    self.skipTest("Base module should not be tested.")

  def _create_example(self):
    """Creates example data for a test"""
    source = np.random.randn(self.batch_size, self.max_decode_length,
                             self.input_depth)
    source_len = np.random.randint(0, self.max_decode_length, [self.batch_size])
    target_len = np.random.randint(0, self.max_decode_length * 2,
                                   [self.batch_size])
    target = np.random.randn(self.batch_size,
                             np.max(target_len), self.input_depth)
    labels = np.random.randint(0, self.vocab_size,
                               [self.batch_size, np.max(target_len) - 1])

    example_ = namedtuple(
        "Example", ["source", "source_len", "target", "target_len", "labels"])
    return example_(source, source_len, target, target_len, labels)

  def test_forward_pass(self):
    """Tests model forward pass by checking the shape of the outputs."""
    ex = self._create_example()
    decoder_input_fn = FixedDecoderInputs(
        inputs=tf.convert_to_tensor(
            ex.target, dtype=tf.float32),
        sequence_length=tf.convert_to_tensor(
            ex.target_len, dtype=tf.int32))

    model = self.create_model()
    decoder_output = model.encode_decode(
        source=tf.convert_to_tensor(
            ex.source, dtype=tf.float32),
        source_len=tf.convert_to_tensor(
            ex.source_len, dtype=tf.int32),
        decoder_input_fn=decoder_input_fn,
        target_len=tf.convert_to_tensor(
            ex.target_len, dtype=tf.int32))

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      decoder_output_ = sess.run(decoder_output)

    max_decode_length = model.params["target.max_seq_len"]
    expected_decode_len = np.minimum(ex.target_len, max_decode_length)

    # Assert shapes are correct
    np.testing.assert_array_equal(decoder_output_.logits.shape, [
        np.max(expected_decode_len), self.batch_size,
        model.target_vocab_info.total_size
    ])
    np.testing.assert_array_equal(
        decoder_output_.predicted_ids.shape,
        [np.max(expected_decode_len), self.batch_size])

  def test_inference(self):
    """Tests model inference by feeding dynamic inputs based on an embedding
    """
    model = self.create_model()
    ex = self._create_example()

    embeddings = tf.get_variable(
        "W_embed", [model.target_vocab_info.total_size, self.input_depth])

    def make_input_fn(predicted_ids):
      """Looks up the predictions in the embeddings.
      """
      return tf.nn.embedding_lookup(embeddings, predicted_ids)

    decoder_input_fn = DynamicDecoderInputs(
        initial_inputs=tf.zeros(
            [self.batch_size, self.input_depth], dtype=tf.float32),
        make_input_fn=make_input_fn)

    decoder_output = model.encode_decode(
        source=tf.convert_to_tensor(
            ex.source, dtype=tf.float32),
        source_len=tf.convert_to_tensor(
            ex.source_len, dtype=tf.int32),
        decoder_input_fn=decoder_input_fn,
        target_len=self.max_decode_length)

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      decoder_output_ = sess.run(decoder_output)

    # Assert shapes are correct
    np.testing.assert_array_equal(decoder_output_.logits.shape, [
        self.max_decode_length, self.batch_size,
        model.target_vocab_info.total_size
    ])
    np.testing.assert_array_equal(decoder_output_.predicted_ids.shape,
                                  [self.max_decode_length, self.batch_size])

  def test_inference_with_beam_search(self):
    """Tests model inference by feeding dynamic inputs based on an embedding
      and using beam search to decode
    """
    self.batch_size = 1
    ex = self._create_example()

    beam_width = 10
    model = self.create_model({"inference.beam_search.beam_width": beam_width})

    embeddings = tf.get_variable(
        "W_embed", [model.target_vocab_info.total_size, self.input_depth])

    def make_input_fn(predicted_ids):
      """Looks up the predictions in the embeddings.
      """
      return tf.nn.embedding_lookup(embeddings, predicted_ids)

    decoder_input_fn = DynamicDecoderInputs(
        initial_inputs=tf.zeros(
            [self.batch_size, self.input_depth], dtype=tf.float32),
        make_input_fn=make_input_fn)

    decoder_output = model.encode_decode(
        source=tf.convert_to_tensor(
            ex.source, dtype=tf.float32),
        source_len=tf.convert_to_tensor(
            ex.source_len, dtype=tf.int32),
        decoder_input_fn=decoder_input_fn,
        target_len=self.max_decode_length)

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      decoder_output_ = sess.run(decoder_output)

    # Assert shapes are correct
    np.testing.assert_array_equal(
        decoder_output_.logits.shape,
        [self.max_decode_length, 1, beam_width,
         model.target_vocab_info.total_size])
    np.testing.assert_array_equal(
        decoder_output_.predicted_ids.shape,
        [self.max_decode_length, 1, beam_width])
    np.testing.assert_array_equal(
        decoder_output_.beam_parent_ids.shape,
        [self.max_decode_length, 1, beam_width])
    np.testing.assert_array_equal(
        decoder_output_.scores.shape,
        [self.max_decode_length, 1, beam_width])
    np.testing.assert_array_equal(
        decoder_output_.original_outputs.predicted_ids.shape,
        [self.max_decode_length, 1, beam_width])
    np.testing.assert_array_equal(
        decoder_output_.original_outputs.logits.shape,
        [self.max_decode_length, 1, beam_width,
         model.target_vocab_info.total_size])

  def test_gradients(self):
    """Ensures the parameter gradients can be computed and are not NaN
    """
    ex = self._create_example()
    decoder_input_fn = FixedDecoderInputs(
        inputs=tf.convert_to_tensor(
            ex.target, dtype=tf.float32),
        sequence_length=tf.convert_to_tensor(
            ex.target_len, dtype=tf.int32))

    model = self.create_model()
    decoder_output = model.encode_decode(
        source=tf.convert_to_tensor(
            ex.source, dtype=tf.float32),
        source_len=tf.convert_to_tensor(
            ex.source_len, dtype=tf.int32),
        decoder_input_fn=decoder_input_fn,
        target_len=tf.convert_to_tensor(
            ex.target_len, dtype=tf.int32))

    # Get a loss to optimize
    losses = seq2seq_losses.cross_entropy_sequence_loss(
        logits=decoder_output.logits,
        targets=tf.ones_like(decoder_output.predicted_ids),
        sequence_length=tf.convert_to_tensor(
            ex.target_len, dtype=tf.int32))
    mean_loss = tf.reduce_mean(losses)

    optimizer = tf.train.AdamOptimizer()
    grads_and_vars = optimizer.compute_gradients(mean_loss)
    train_op = optimizer.apply_gradients(grads_and_vars)

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      _, grads_and_vars_ = sess.run([train_op, grads_and_vars])

    for grad, _ in grads_and_vars_:
      self.assertFalse(np.isnan(grad).any())

  def _test_pipeline(self, mode, params=None):
    """Helper function to test the full model pipeline.
    """
    # Create source and target example
    source_len = 10
    target_len = self.max_decode_length + 10
    source = " ".join(np.random.choice(self.vocab_list, source_len))
    target = " ".join(np.random.choice(self.vocab_list, target_len))
    sources_file, targets_file = test_utils.create_temp_parallel_data(
        sources=[source], targets=[target])

    # Build model graph
    model = self.create_model(params)
    data_provider = lambda: data_utils.make_parallel_data_provider(
        [sources_file.name], [targets_file.name])
    input_fn = training_utils.create_input_fn(data_provider, self.batch_size)
    features, labels = input_fn()
    fetches = model(features, labels, None, mode)
    fetches = [_ for _ in fetches if _ is not None]

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      sess.run(tf.local_variables_initializer())
      sess.run(tf.initialize_all_tables())
      with tf.contrib.slim.queues.QueueRunners(sess):
        fetches_ = sess.run(fetches)

    sources_file.close()
    targets_file.close()

    return model, fetches_

  def test_pipeline_train(self):
    model, fetches_ = self._test_pipeline(tf.contrib.learn.ModeKeys.TRAIN)
    predictions_, loss_, _ = fetches_

    # We have predictions for each target words and the SEQUENCE_START token.
    # That's why it's `target_len + 1`
    target_len = self.max_decode_length + 10
    max_decode_length = model.params["target.max_seq_len"]
    expected_decode_len = np.minimum(target_len + 1, max_decode_length)

    np.testing.assert_array_equal(
        predictions_["logits"].shape,
        [self.batch_size, expected_decode_len,
         model.target_vocab_info.total_size])
    np.testing.assert_array_equal(
        predictions_["losses"].shape,
        [self.batch_size, expected_decode_len - 1])
    np.testing.assert_array_equal(
        predictions_["predicted_ids"].shape,
        [self.batch_size, expected_decode_len])
    self.assertFalse(np.isnan(loss_))

  def test_pipeline_inference(self):
    model, fetches_ = self._test_pipeline(tf.contrib.learn.ModeKeys.INFER)
    predictions_, = fetches_
    np.testing.assert_array_equal(
        predictions_["logits"].shape,
        [self.batch_size, model.params["target.max_seq_len"],
         model.target_vocab_info.total_size])
    np.testing.assert_array_equal(
        predictions_["predicted_ids"].shape,
        [self.batch_size, model.params["target.max_seq_len"]])

  def test_pipeline_beam_search_infer(self):
    self.batch_size = 1
    beam_width = 10
    model, fetches_ = self._test_pipeline(
        mode=tf.contrib.learn.ModeKeys.INFER,
        params={"inference.beam_search.beam_width": 10})
    predictions_, = fetches_

    seq_length = model.params["target.max_seq_len"]
    vocab_size = model.target_vocab_info.total_size
    np.testing.assert_array_equal(
        predictions_["logits"].shape,
        [1, seq_length, beam_width, vocab_size])
    np.testing.assert_array_equal(
        predictions_["predicted_ids"].shape,
        [1, seq_length, beam_width])
    np.testing.assert_array_equal(
        predictions_["beam_parent_ids"].shape,
        [1, seq_length, beam_width])
    np.testing.assert_array_equal(
        predictions_["scores"].shape,
        [1, seq_length, beam_width])
    np.testing.assert_array_equal(
        predictions_["original_outputs.predicted_ids"].shape,
        [1, seq_length, beam_width])
    np.testing.assert_array_equal(
        predictions_["original_outputs.logits"].shape,
        [1, seq_length, beam_width, vocab_size])

class TestBasicSeq2Seq(EncoderDecoderTests):
  """Tests the seq2seq.models.BasicSeq2Seq model.
  """

  def setUp(self):
    super(TestBasicSeq2Seq, self).setUp()

  def create_model(self, params=None):
    params_ = BasicSeq2Seq.default_params().copy()
    params_.update({
        "rnn_cell.dropout_input_keep_prob": 0.8,
        "rnn_cell.cell_spec":  """
            { "class": "LSTMCell", "num_units": 128 }"""
    })
    params_.update(params or {})
    return BasicSeq2Seq(
        source_vocab_info=self.vocab_info,
        target_vocab_info=self.vocab_info,
        params=params_)

class TestAttentionSeq2Seq(EncoderDecoderTests):
  """Tests the seq2seq.models.AttentionSeq2Seq model.
  """

  def setUp(self):
    super(TestAttentionSeq2Seq, self).setUp()
    self.encoder_rnn_cell = tf.contrib.rnn.LSTMCell(32)
    self.decoder_rnn_cell = tf.contrib.rnn.LSTMCell(32)
    self.attention_dim = 128

  def create_model(self, params=None):
    params_ = AttentionSeq2Seq.default_params().copy()
    params_.update({
        "source.reverse": True
    })
    params_.update(params or {})
    return AttentionSeq2Seq(
        source_vocab_info=self.vocab_info,
        target_vocab_info=self.vocab_info,
        params=params_)

if __name__ == "__main__":
  tf.test.main()
<commit_msg>Add back num_layers for test<commit_after># -*- coding: utf-8 -*-

"""Tests for Models
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from collections import namedtuple

from seq2seq import losses as seq2seq_losses
from seq2seq.data import data_utils, vocab
from seq2seq.training import utils as training_utils
from seq2seq.test import utils as test_utils
from seq2seq.models import BasicSeq2Seq, AttentionSeq2Seq
from seq2seq.decoders import FixedDecoderInputs, DynamicDecoderInputs

import tensorflow as tf
import numpy as np

class EncoderDecoderTests(tf.test.TestCase):
  """Base class for EncoderDecoder tests. Tests for specific classes should
  inherit from this and tf.test.TestCase.
  """

  def setUp(self):
    super(EncoderDecoderTests, self).setUp()
    tf.logging.set_verbosity(tf.logging.INFO)
    self.batch_size = 4
    self.input_depth = 32
    self.max_decode_length = 40

    # Create vocabulary
    self.vocab_list = [str(_) for _ in range(10)]
    self.vocab_list += ["笑う", "泣く", "了解", "はい", "＾＿＾"]
    self.vocab_size = len(self.vocab_list)
    self.vocab_file = test_utils.create_temporary_vocab_file(self.vocab_list)
    self.vocab_info = vocab.get_vocab_info(self.vocab_file.name)

  def tearDown(self):
    self.vocab_file.close()

  def create_model(self, _params=None):
    """Creates model class to be tested. Subclasses must implement this method.
    """
    self.skipTest("Base module should not be tested.")

  def _create_example(self):
    """Creates example data for a test"""
    source = np.random.randn(self.batch_size, self.max_decode_length,
                             self.input_depth)
    source_len = np.random.randint(0, self.max_decode_length, [self.batch_size])
    target_len = np.random.randint(0, self.max_decode_length * 2,
                                   [self.batch_size])
    target = np.random.randn(self.batch_size,
                             np.max(target_len), self.input_depth)
    labels = np.random.randint(0, self.vocab_size,
                               [self.batch_size, np.max(target_len) - 1])

    example_ = namedtuple(
        "Example", ["source", "source_len", "target", "target_len", "labels"])
    return example_(source, source_len, target, target_len, labels)

  def test_forward_pass(self):
    """Tests model forward pass by checking the shape of the outputs."""
    ex = self._create_example()
    decoder_input_fn = FixedDecoderInputs(
        inputs=tf.convert_to_tensor(
            ex.target, dtype=tf.float32),
        sequence_length=tf.convert_to_tensor(
            ex.target_len, dtype=tf.int32))

    model = self.create_model()
    decoder_output = model.encode_decode(
        source=tf.convert_to_tensor(
            ex.source, dtype=tf.float32),
        source_len=tf.convert_to_tensor(
            ex.source_len, dtype=tf.int32),
        decoder_input_fn=decoder_input_fn,
        target_len=tf.convert_to_tensor(
            ex.target_len, dtype=tf.int32))

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      decoder_output_ = sess.run(decoder_output)

    max_decode_length = model.params["target.max_seq_len"]
    expected_decode_len = np.minimum(ex.target_len, max_decode_length)

    # Assert shapes are correct
    np.testing.assert_array_equal(decoder_output_.logits.shape, [
        np.max(expected_decode_len), self.batch_size,
        model.target_vocab_info.total_size
    ])
    np.testing.assert_array_equal(
        decoder_output_.predicted_ids.shape,
        [np.max(expected_decode_len), self.batch_size])

  def test_inference(self):
    """Tests model inference by feeding dynamic inputs based on an embedding
    """
    model = self.create_model()
    ex = self._create_example()

    embeddings = tf.get_variable(
        "W_embed", [model.target_vocab_info.total_size, self.input_depth])

    def make_input_fn(predicted_ids):
      """Looks up the predictions in the embeddings.
      """
      return tf.nn.embedding_lookup(embeddings, predicted_ids)

    decoder_input_fn = DynamicDecoderInputs(
        initial_inputs=tf.zeros(
            [self.batch_size, self.input_depth], dtype=tf.float32),
        make_input_fn=make_input_fn)

    decoder_output = model.encode_decode(
        source=tf.convert_to_tensor(
            ex.source, dtype=tf.float32),
        source_len=tf.convert_to_tensor(
            ex.source_len, dtype=tf.int32),
        decoder_input_fn=decoder_input_fn,
        target_len=self.max_decode_length)

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      decoder_output_ = sess.run(decoder_output)

    # Assert shapes are correct
    np.testing.assert_array_equal(decoder_output_.logits.shape, [
        self.max_decode_length, self.batch_size,
        model.target_vocab_info.total_size
    ])
    np.testing.assert_array_equal(decoder_output_.predicted_ids.shape,
                                  [self.max_decode_length, self.batch_size])

  def test_inference_with_beam_search(self):
    """Tests model inference by feeding dynamic inputs based on an embedding
      and using beam search to decode
    """
    self.batch_size = 1
    ex = self._create_example()

    beam_width = 10
    model = self.create_model({"inference.beam_search.beam_width": beam_width})

    embeddings = tf.get_variable(
        "W_embed", [model.target_vocab_info.total_size, self.input_depth])

    def make_input_fn(predicted_ids):
      """Looks up the predictions in the embeddings.
      """
      return tf.nn.embedding_lookup(embeddings, predicted_ids)

    decoder_input_fn = DynamicDecoderInputs(
        initial_inputs=tf.zeros(
            [self.batch_size, self.input_depth], dtype=tf.float32),
        make_input_fn=make_input_fn)

    decoder_output = model.encode_decode(
        source=tf.convert_to_tensor(
            ex.source, dtype=tf.float32),
        source_len=tf.convert_to_tensor(
            ex.source_len, dtype=tf.int32),
        decoder_input_fn=decoder_input_fn,
        target_len=self.max_decode_length)

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      decoder_output_ = sess.run(decoder_output)

    # Assert shapes are correct
    np.testing.assert_array_equal(
        decoder_output_.logits.shape,
        [self.max_decode_length, 1, beam_width,
         model.target_vocab_info.total_size])
    np.testing.assert_array_equal(
        decoder_output_.predicted_ids.shape,
        [self.max_decode_length, 1, beam_width])
    np.testing.assert_array_equal(
        decoder_output_.beam_parent_ids.shape,
        [self.max_decode_length, 1, beam_width])
    np.testing.assert_array_equal(
        decoder_output_.scores.shape,
        [self.max_decode_length, 1, beam_width])
    np.testing.assert_array_equal(
        decoder_output_.original_outputs.predicted_ids.shape,
        [self.max_decode_length, 1, beam_width])
    np.testing.assert_array_equal(
        decoder_output_.original_outputs.logits.shape,
        [self.max_decode_length, 1, beam_width,
         model.target_vocab_info.total_size])

  def test_gradients(self):
    """Ensures the parameter gradients can be computed and are not NaN
    """
    ex = self._create_example()
    decoder_input_fn = FixedDecoderInputs(
        inputs=tf.convert_to_tensor(
            ex.target, dtype=tf.float32),
        sequence_length=tf.convert_to_tensor(
            ex.target_len, dtype=tf.int32))

    model = self.create_model()
    decoder_output = model.encode_decode(
        source=tf.convert_to_tensor(
            ex.source, dtype=tf.float32),
        source_len=tf.convert_to_tensor(
            ex.source_len, dtype=tf.int32),
        decoder_input_fn=decoder_input_fn,
        target_len=tf.convert_to_tensor(
            ex.target_len, dtype=tf.int32))

    # Get a loss to optimize
    losses = seq2seq_losses.cross_entropy_sequence_loss(
        logits=decoder_output.logits,
        targets=tf.ones_like(decoder_output.predicted_ids),
        sequence_length=tf.convert_to_tensor(
            ex.target_len, dtype=tf.int32))
    mean_loss = tf.reduce_mean(losses)

    optimizer = tf.train.AdamOptimizer()
    grads_and_vars = optimizer.compute_gradients(mean_loss)
    train_op = optimizer.apply_gradients(grads_and_vars)

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      _, grads_and_vars_ = sess.run([train_op, grads_and_vars])

    for grad, _ in grads_and_vars_:
      self.assertFalse(np.isnan(grad).any())

  def _test_pipeline(self, mode, params=None):
    """Helper function to test the full model pipeline.
    """
    # Create source and target example
    source_len = 10
    target_len = self.max_decode_length + 10
    source = " ".join(np.random.choice(self.vocab_list, source_len))
    target = " ".join(np.random.choice(self.vocab_list, target_len))
    sources_file, targets_file = test_utils.create_temp_parallel_data(
        sources=[source], targets=[target])

    # Build model graph
    model = self.create_model(params)
    data_provider = lambda: data_utils.make_parallel_data_provider(
        [sources_file.name], [targets_file.name])
    input_fn = training_utils.create_input_fn(data_provider, self.batch_size)
    features, labels = input_fn()
    fetches = model(features, labels, None, mode)
    fetches = [_ for _ in fetches if _ is not None]

    with self.test_session() as sess:
      sess.run(tf.global_variables_initializer())
      sess.run(tf.local_variables_initializer())
      sess.run(tf.initialize_all_tables())
      with tf.contrib.slim.queues.QueueRunners(sess):
        fetches_ = sess.run(fetches)

    sources_file.close()
    targets_file.close()

    return model, fetches_

  def test_pipeline_train(self):
    model, fetches_ = self._test_pipeline(tf.contrib.learn.ModeKeys.TRAIN)
    predictions_, loss_, _ = fetches_

    # We have predictions for each target words and the SEQUENCE_START token.
    # That's why it's `target_len + 1`
    target_len = self.max_decode_length + 10
    max_decode_length = model.params["target.max_seq_len"]
    expected_decode_len = np.minimum(target_len + 1, max_decode_length)

    np.testing.assert_array_equal(
        predictions_["logits"].shape,
        [self.batch_size, expected_decode_len,
         model.target_vocab_info.total_size])
    np.testing.assert_array_equal(
        predictions_["losses"].shape,
        [self.batch_size, expected_decode_len - 1])
    np.testing.assert_array_equal(
        predictions_["predicted_ids"].shape,
        [self.batch_size, expected_decode_len])
    self.assertFalse(np.isnan(loss_))

  def test_pipeline_inference(self):
    model, fetches_ = self._test_pipeline(tf.contrib.learn.ModeKeys.INFER)
    predictions_, = fetches_
    np.testing.assert_array_equal(
        predictions_["logits"].shape,
        [self.batch_size, model.params["target.max_seq_len"],
         model.target_vocab_info.total_size])
    np.testing.assert_array_equal(
        predictions_["predicted_ids"].shape,
        [self.batch_size, model.params["target.max_seq_len"]])

  def test_pipeline_beam_search_infer(self):
    self.batch_size = 1
    beam_width = 10
    model, fetches_ = self._test_pipeline(
        mode=tf.contrib.learn.ModeKeys.INFER,
        params={"inference.beam_search.beam_width": 10})
    predictions_, = fetches_

    seq_length = model.params["target.max_seq_len"]
    vocab_size = model.target_vocab_info.total_size
    np.testing.assert_array_equal(
        predictions_["logits"].shape,
        [1, seq_length, beam_width, vocab_size])
    np.testing.assert_array_equal(
        predictions_["predicted_ids"].shape,
        [1, seq_length, beam_width])
    np.testing.assert_array_equal(
        predictions_["beam_parent_ids"].shape,
        [1, seq_length, beam_width])
    np.testing.assert_array_equal(
        predictions_["scores"].shape,
        [1, seq_length, beam_width])
    np.testing.assert_array_equal(
        predictions_["original_outputs.predicted_ids"].shape,
        [1, seq_length, beam_width])
    np.testing.assert_array_equal(
        predictions_["original_outputs.logits"].shape,
        [1, seq_length, beam_width, vocab_size])

class TestBasicSeq2Seq(EncoderDecoderTests):
  """Tests the seq2seq.models.BasicSeq2Seq model.
  """

  def setUp(self):
    super(TestBasicSeq2Seq, self).setUp()

  def create_model(self, params=None):
    params_ = BasicSeq2Seq.default_params().copy()
    params_.update({
        "rnn_cell.dropout_input_keep_prob": 0.8,
        "rnn_cell.num_layers": 2,
        "rnn_cell.cell_spec":  """
            { "class": "LSTMCell", "num_units": 128 }"""
    })
    params_.update(params or {})
    return BasicSeq2Seq(
        source_vocab_info=self.vocab_info,
        target_vocab_info=self.vocab_info,
        params=params_)

class TestAttentionSeq2Seq(EncoderDecoderTests):
  """Tests the seq2seq.models.AttentionSeq2Seq model.
  """

  def setUp(self):
    super(TestAttentionSeq2Seq, self).setUp()
    self.encoder_rnn_cell = tf.contrib.rnn.LSTMCell(32)
    self.decoder_rnn_cell = tf.contrib.rnn.LSTMCell(32)
    self.attention_dim = 128

  def create_model(self, params=None):
    params_ = AttentionSeq2Seq.default_params().copy()
    params_.update({
        "source.reverse": True
    })
    params_.update(params or {})
    return AttentionSeq2Seq(
        source_vocab_info=self.vocab_info,
        target_vocab_info=self.vocab_info,
        params=params_)

if __name__ == "__main__":
  tf.test.main()
<|endoftext|>


<commit_before>import os
import sys
import errno
import uuid

from atomicwrites import atomic_write

__version__ = '0.1.0'

PY2 = sys.version_info[0] == 2

def to_unicode(x, encoding='ascii'):
    if not isinstance(x, text_type):
        return x.decode(encoding)
    return x

def to_bytes(x, encoding='ascii'):
    if not isinstance(x, bytes):
        return x.encode(encoding)
    return x

if PY2:
    text_type = unicode  # noqa
    to_native = to_bytes

else:
    text_type = str  # noqa
    to_native = to_unicode

SAFE_UID_CHARS = ('abcdefghijklmnopqrstuvwxyz'
                  'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
                  '0123456789_.-+')

def _href_safe(ident, safe=SAFE_UID_CHARS):
    return not bool(set(ident) - set(safe))

def _generate_href(ident=None, safe=SAFE_UID_CHARS):
    if not ident or not _href_safe(ident, safe):
        return to_unicode(uuid.uuid4().hex)
    else:
        return ident

def _get_etag_from_file(fpath):
    '''Get mtime-based etag from a filepath.'''
    stat = os.stat(fpath)
    mtime = getattr(stat, 'st_mtime_ns', None)
    if mtime is None:
        mtime = stat.st_mtime
    return '{:.9f}'.format(mtime)

class VdirError(IOError):
    pass

class NotFoundError(VdirError):
    pass

class WrongEtagError(VdirError):
    pass

class AlreadyExists(VdirError):
    pass

class Item(object):
    def __init__(self, raw):
        assert isinstance(raw, text_type)
        self.raw = raw

def _normalize_meta_value(value):
    return to_native(value or u'').strip()

class VdirBase(object):
    item_class = Item
    default_mode = 0o750

    def __init__(self, path, fileext, encoding='utf-8'):
        self.path = path
        self.encoding = encoding
        self.fileext = fileext

    @classmethod
    def discover(cls, path, **kwargs):
        try:
            collections = os.listdir(path)
        except OSError as e:
            if e.errno != errno.ENOENT:
                raise
            return

        for collection in collections:
            collection_path = os.path.join(path, collection)
            if os.path.isdir(collection_path):
                yield cls(path=collection_path, **kwargs)

    @classmethod
    def create(cls, collection_name, **kwargs):
        kwargs = dict(kwargs)
        path = kwargs['path']

        path = os.path.join(path, collection_name)
        if not os.path.exists(path):
            os.makedirs(path, mode=cls.default_mode)
        elif not os.path.isdir(path):
            raise IOError('{} is not a directory.'.format(repr(path)))

        kwargs['path'] = path
        return kwargs

    def _get_filepath(self, href):
        return os.path.join(self.path, href)

    def _get_href(self, ident):
        return _generate_href(ident) + self.fileext

    def list(self):
        for fname in os.listdir(self.path):
            fpath = os.path.join(self.path, fname)
            if os.path.isfile(fpath) and fname.endswith(self.fileext):
                yield fname, _get_etag_from_file(fpath)

    def get(self, href):
        fpath = self._get_filepath(href)
        try:
            with open(fpath, 'rb') as f:
                return (Item(f.read().decode(self.encoding)),
                        _get_etag_from_file(fpath))
        except IOError as e:
            if e.errno == errno.ENOENT:
                raise NotFoundError(href)
            else:
                raise

    def upload(self, item):
        if not isinstance(item.raw, text_type):
            raise TypeError('item.raw must be a unicode string.')

        try:
            href = self._get_href(item.ident)
            fpath, etag = self._upload_impl(item, href)
        except OSError as e:
            if e.errno in (
                errno.ENAMETOOLONG,  # Unix
                errno.ENOENT  # Windows
            ):
                # random href instead of UID-based
                href = self._get_href(None)
                fpath, etag = self._upload_impl(item, href)
            else:
                raise

        if self.post_hook:
            self._run_post_hook(fpath)
        return href, etag

    def _upload_impl(self, item, href):
        fpath = self._get_filepath(href)
        try:
            with atomic_write(fpath, mode='wb', overwrite=False) as f:
                f.write(item.raw.encode(self.encoding))
                return fpath, _get_etag_from_file(f.name)
        except OSError as e:
            if e.errno == errno.EEXIST:
                raise AlreadyExists(existing_href=href)
            else:
                raise

    def update(self, href, item, etag):
        fpath = self._get_filepath(href)
        if not os.path.exists(fpath):
            raise NotFoundError(item.uid)
        actual_etag = _get_etag_from_file(fpath)
        if etag != actual_etag:
            raise WrongEtagError(etag, actual_etag)

        if not isinstance(item.raw, text_type):
            raise TypeError('item.raw must be a unicode string.')

        with atomic_write(fpath, mode='wb', overwrite=True) as f:
            f.write(item.raw.encode(self.encoding))
            etag = _get_etag_from_fileobject(f)

        return etag

    def delete(self, href, etag):
        fpath = self._get_filepath(href)
        if not os.path.isfile(fpath):
            raise NotFoundError(href)
        actual_etag = _get_etag_from_file(fpath)
        if etag != actual_etag:
            raise WrongEtagError(etag, actual_etag)
        os.remove(fpath)

    def get_meta(self, key):
        fpath = os.path.join(self.path, key)
        try:
            with open(fpath, 'rb') as f:
                return f.read().decode(self.encoding) or None
        except IOError as e:
            if e.errno == errno.ENOENT:
                return None
            else:
                raise

    def set_meta(self, key, value):
        value = value or u''
        assert isinstance(value, text_type)
        fpath = os.path.join(self.path, key)
        with atomic_write(fpath, mode='wb', overwrite=True) as f:
            f.write(value.encode(self.encoding))

class cached_property(object):
    '''A read-only @property that is only evaluated once. Only usable on class
    instances' methods.
    '''
    def __init__(self, fget, doc=None):
        self.__name__ = fget.__name__
        self.__module__ = fget.__module__
        self.__doc__ = doc or fget.__doc__
        self.fget = fget

    def __get__(self, obj, cls):
        if obj is None:  # pragma: no cover
            return self
        obj.__dict__[self.__name__] = result = self.fget(obj)
        return result

class Color(object):
    def __init__(self, x):
        if not x:
            raise ValueError('Color is false-ish.')
        if not x.startswith('#'):
            raise ValueError('Color must start with a #.')
        if len(x) != 7:
            raise ValueError('Color must not have shortcuts. '
                             '#ffffff instead of #fff')
        self.raw = x.upper()

    @cached_property
    def rgb():
        x = self.raw

        r = x[1:3]
        g = x[3:5]
        b = x[5:8]

        if len(r) == len(g) == len(b) == 2:
            return int(r, 16), int(g, 16), int(b, 16)
        else:
            raise ValueError('Unable to parse color value: {}'
                             .format(self.value)

class ColorMixin(object):
    color_type = Color

    def get_color(self):
        try:
            return self.color_type(self.get_meta('color'))
        except ValueError:
            return None

    def set_color(self, value):
        self.set_meta('color', self.color_type(value).raw)

class DisplayNameMixin(object):
    def get_displayname(self):
        return self.get_meta('displayname')

    def set_displayname(self, value):
        self.set_meta('displayname', value)

class Vdir(VdirBase, ColorMixin, DisplayNameMixin):
    pass
<commit_msg>Fix syntax<commit_after>import os
import sys
import errno
import uuid

from atomicwrites import atomic_write

__version__ = '0.1.0'

PY2 = sys.version_info[0] == 2

def to_unicode(x, encoding='ascii'):
    if not isinstance(x, text_type):
        return x.decode(encoding)
    return x

def to_bytes(x, encoding='ascii'):
    if not isinstance(x, bytes):
        return x.encode(encoding)
    return x

if PY2:
    text_type = unicode  # noqa
    to_native = to_bytes

else:
    text_type = str  # noqa
    to_native = to_unicode

SAFE_UID_CHARS = ('abcdefghijklmnopqrstuvwxyz'
                  'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
                  '0123456789_.-+')

def _href_safe(ident, safe=SAFE_UID_CHARS):
    return not bool(set(ident) - set(safe))

def _generate_href(ident=None, safe=SAFE_UID_CHARS):
    if not ident or not _href_safe(ident, safe):
        return to_unicode(uuid.uuid4().hex)
    else:
        return ident

def _get_etag_from_file(fpath):
    '''Get mtime-based etag from a filepath.'''
    stat = os.stat(fpath)
    mtime = getattr(stat, 'st_mtime_ns', None)
    if mtime is None:
        mtime = stat.st_mtime
    return '{:.9f}'.format(mtime)

class VdirError(IOError):
    pass

class NotFoundError(VdirError):
    pass

class WrongEtagError(VdirError):
    pass

class AlreadyExists(VdirError):
    pass

class Item(object):
    def __init__(self, raw):
        assert isinstance(raw, text_type)
        self.raw = raw

def _normalize_meta_value(value):
    return to_native(value or u'').strip()

class VdirBase(object):
    item_class = Item
    default_mode = 0o750

    def __init__(self, path, fileext, encoding='utf-8'):
        self.path = path
        self.encoding = encoding
        self.fileext = fileext

    @classmethod
    def discover(cls, path, **kwargs):
        try:
            collections = os.listdir(path)
        except OSError as e:
            if e.errno != errno.ENOENT:
                raise
            return

        for collection in collections:
            collection_path = os.path.join(path, collection)
            if os.path.isdir(collection_path):
                yield cls(path=collection_path, **kwargs)

    @classmethod
    def create(cls, collection_name, **kwargs):
        kwargs = dict(kwargs)
        path = kwargs['path']

        path = os.path.join(path, collection_name)
        if not os.path.exists(path):
            os.makedirs(path, mode=cls.default_mode)
        elif not os.path.isdir(path):
            raise IOError('{} is not a directory.'.format(repr(path)))

        kwargs['path'] = path
        return kwargs

    def _get_filepath(self, href):
        return os.path.join(self.path, href)

    def _get_href(self, ident):
        return _generate_href(ident) + self.fileext

    def list(self):
        for fname in os.listdir(self.path):
            fpath = os.path.join(self.path, fname)
            if os.path.isfile(fpath) and fname.endswith(self.fileext):
                yield fname, _get_etag_from_file(fpath)

    def get(self, href):
        fpath = self._get_filepath(href)
        try:
            with open(fpath, 'rb') as f:
                return (Item(f.read().decode(self.encoding)),
                        _get_etag_from_file(fpath))
        except IOError as e:
            if e.errno == errno.ENOENT:
                raise NotFoundError(href)
            else:
                raise

    def upload(self, item):
        if not isinstance(item.raw, text_type):
            raise TypeError('item.raw must be a unicode string.')

        try:
            href = self._get_href(item.ident)
            fpath, etag = self._upload_impl(item, href)
        except OSError as e:
            if e.errno in (
                errno.ENAMETOOLONG,  # Unix
                errno.ENOENT  # Windows
            ):
                # random href instead of UID-based
                href = self._get_href(None)
                fpath, etag = self._upload_impl(item, href)
            else:
                raise

        if self.post_hook:
            self._run_post_hook(fpath)
        return href, etag

    def _upload_impl(self, item, href):
        fpath = self._get_filepath(href)
        try:
            with atomic_write(fpath, mode='wb', overwrite=False) as f:
                f.write(item.raw.encode(self.encoding))
                return fpath, _get_etag_from_file(f.name)
        except OSError as e:
            if e.errno == errno.EEXIST:
                raise AlreadyExists(existing_href=href)
            else:
                raise

    def update(self, href, item, etag):
        fpath = self._get_filepath(href)
        if not os.path.exists(fpath):
            raise NotFoundError(item.uid)
        actual_etag = _get_etag_from_file(fpath)
        if etag != actual_etag:
            raise WrongEtagError(etag, actual_etag)

        if not isinstance(item.raw, text_type):
            raise TypeError('item.raw must be a unicode string.')

        with atomic_write(fpath, mode='wb', overwrite=True) as f:
            f.write(item.raw.encode(self.encoding))
            etag = _get_etag_from_fileobject(f)

        return etag

    def delete(self, href, etag):
        fpath = self._get_filepath(href)
        if not os.path.isfile(fpath):
            raise NotFoundError(href)
        actual_etag = _get_etag_from_file(fpath)
        if etag != actual_etag:
            raise WrongEtagError(etag, actual_etag)
        os.remove(fpath)

    def get_meta(self, key):
        fpath = os.path.join(self.path, key)
        try:
            with open(fpath, 'rb') as f:
                return f.read().decode(self.encoding) or None
        except IOError as e:
            if e.errno == errno.ENOENT:
                return None
            else:
                raise

    def set_meta(self, key, value):
        value = value or u''
        assert isinstance(value, text_type)
        fpath = os.path.join(self.path, key)
        with atomic_write(fpath, mode='wb', overwrite=True) as f:
            f.write(value.encode(self.encoding))

class cached_property(object):
    '''A read-only @property that is only evaluated once. Only usable on class
    instances' methods.
    '''
    def __init__(self, fget, doc=None):
        self.__name__ = fget.__name__
        self.__module__ = fget.__module__
        self.__doc__ = doc or fget.__doc__
        self.fget = fget

    def __get__(self, obj, cls):
        if obj is None:  # pragma: no cover
            return self
        obj.__dict__[self.__name__] = result = self.fget(obj)
        return result

class Color(object):
    def __init__(self, x):
        if not x:
            raise ValueError('Color is false-ish.')
        if not x.startswith('#'):
            raise ValueError('Color must start with a #.')
        if len(x) != 7:
            raise ValueError('Color must not have shortcuts. '
                             '#ffffff instead of #fff')
        self.raw = x.upper()

    @cached_property
    def rgb():
        x = self.raw

        r = x[1:3]
        g = x[3:5]
        b = x[5:8]

        if len(r) == len(g) == len(b) == 2:
            return int(r, 16), int(g, 16), int(b, 16)
        else:
            raise ValueError('Unable to parse color value: {}'
                             .format(self.value))

class ColorMixin(object):
    color_type = Color

    def get_color(self):
        try:
            return self.color_type(self.get_meta('color'))
        except ValueError:
            return None

    def set_color(self, value):
        self.set_meta('color', self.color_type(value).raw)

class DisplayNameMixin(object):
    def get_displayname(self):
        return self.get_meta('displayname')

    def set_displayname(self, value):
        self.set_meta('displayname', value)

class Vdir(VdirBase, ColorMixin, DisplayNameMixin):
    pass
<|endoftext|>


<commit_before>import json
import logging
import uuid
import urllib.parse

from django.db import models
from django.utils import timezone
from django.utils.text import slugify
from django.conf import settings
from channels import Group
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.contrib.auth.models import User
from django.core.mail import send_mail

import radio.choices as choice

from pinax.stripe.models import Plan as pinax_Plan

log = logging.getLogger(__name__)

class Agency(models.Model):
    name = models.CharField(max_length=100)
    short = models.CharField(max_length=5, unique=True)

    def __str__(self):
        return self.name

    def get_short(self):
        return self.short

class System(models.Model):
    name = models.CharField(max_length=100)
    system_id = models.CharField(max_length=20, blank=True, null=True)

    def __str__(self):
        return self.name

class Source(models.Model):
    description = models.CharField(max_length=100)

    class Meta:
        ordering = ["pk"]

    def __str__(self):
        return self.description

class Unit(models.Model):
    dec_id = models.IntegerField()
    description = models.CharField(max_length=100, blank=True, null=True)
    agency = models.ForeignKey(Agency, default=settings.RADIO_DEFAULT_UNIT_AGENCY)
    #agency = models.ForeignKey(Agency, default=2)
    type = models.CharField(max_length=1, choices=choice.RADIO_TYPE_CHOICES, default=choice.RADIO_TYPE_MOBILE)
    number = models.IntegerField(default=1)
    system = models.ForeignKey(System, default=0)
    slug = models.SlugField(null=True, blank=True)

    class Meta:
        unique_together = ('dec_id', 'system',)

    def __str__(self):
        if(self.description):
            return self.description
        else:
            return str(self.dec_id)

    def save(self, *args, **kwargs):
        if self.description:
            self.slug = slugify(self.description)
        super(Unit, self).save(*args, **kwargs)

class RepeaterSite(models.Model):
    name = models.CharField(max_length=100, unique=True)
    description = models.CharField(max_length=500, blank=True, null=True)

    def __str__(self):
        return self.name

class Service(models.Model):
    name = models.CharField(max_length=100, unique=True)
    description = models.CharField(max_length=500, blank=True, null=True)

    def __str__(self):
        return self.name

class TalkGroup(models.Model):
    dec_id = models.IntegerField()
    alpha_tag = models.CharField(max_length=30)
    common_name = models.CharField(max_length=10, blank=True, null=True)
    description = models.CharField(max_length=100, blank=True, null=True)
    slug = models.SlugField(null=True)
    public = models.BooleanField(default=True)
    comments = models.CharField(max_length=100, blank=True, null=True)
    system = models.ForeignKey(System, default=0)
    mode = models.CharField(max_length=1, choices=choice.TG_MODE_CHOICES, default=choice.TG_MODE_DIGITAL, help_text='mode used by trunk-recorder')
    priority = models.IntegerField(default=3, help_text='record priority used by trunk-recorder')
    _home_site = models.ForeignKey(RepeaterSite, blank=True, null=True)
    _service_type = models.ForeignKey(Service, blank=True, null=True)

    class Meta:
        ordering = ["alpha_tag"]
        unique_together = ('dec_id', 'system',)

    def __str__(self):
        return self.alpha_tag

    def save(self, *args, **kwargs):
        self.slug = slugify(self.alpha_tag)
        super(TalkGroup, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return '/tg/{}/'.format(self.slug)

    @property
    def home_site(self):
        return self._home_site

    @home_site.setter
    def home_site(self, value):
        hs, created = RepeaterSite.objects.get_or_create(name=value)
        self._home_site = hs

    @property
    def service_type(self):
        return self._service_type

    @service_type.setter
    def service_type(self, value):
        st, created = Service.objects.get_or_create(name=value)
        self._service_type = st

def AddToDefaultAccessGroup(sender, instance, created, **kwargs):
    if created:
        new_tg = TalkGroupWithSystem.objects.get(pk=instance.pk)
        for access_group in TalkGroupAccess.objects.filter(default_new_talkgroups=True):
            access_group.talkgroups.add(new_tg)
            access_group.save()

post_save.connect(AddToDefaultAccessGroup, sender=TalkGroup)

class TalkGroupWithSystem(TalkGroup):
    class Meta:
        proxy = True

    def __str__(self):
        return "{} ({})".format(self.alpha_tag, self.system)

class Transmission(models.Model):
    slug = models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)
    start_datetime = models.DateTimeField(db_index=True)
    audio_file = models.FileField()
    audio_file_url_path = models.CharField(max_length=100, default='/')
    talkgroup = models.IntegerField()
    talkgroup_info = models.ForeignKey(TalkGroup)
    freq = models.IntegerField()
    emergency = models.BooleanField(default=False)
    units = models.ManyToManyField(Unit, through='TranmissionUnit')
    play_length = models.FloatField(default=0.0)
    source = models.ForeignKey(Source, default=0)
    system = models.ForeignKey(System, default=0)

    def __str__(self):
        return '{} {}'.format(self.talkgroup, self.start_datetime)

    def get_absolute_url(self):
        return '/audio/{}/'.format(self.slug)

    @property
    def local_start_datetime(self):
        return timezone.localtime(self.start_datetime).strftime('%H:%M:%S %m/%d/%Y')

    def as_dict(self):
        return {'start_datetime': str(self.start_datetime), 'audio_file': str(self.audio_file), 'talkgroup_desc': self.talkgroup_info.alpha_tag}

    def print_play_length(self):
        m, s = divmod(int(self.play_length), 60)
        return '{:02d}:{:02d}'.format(m,s)

    def freq_mhz(self):
        return '{0:07.3f}'.format(self.freq / 1000000)

    def tg_name(self):
        """Returns TalkGroup name used for page title
           First tries TalkGroup.common_name
           then TalkGroup.alpha_tag
        """
        if self.talkgroup_info.common_name:
            return self.talkgroup_info.common_name
        else:
            return self.talkgroup_info.alpha_tag

    @property
    def audio_url(self):
        base_path = settings.AUDIO_URL_BASE
        return urllib.parse.urljoin(base_path, self.audio_file_url_path.lstrip('/'))

    class Meta:
        ordering = ["-pk"]

@receiver(post_save, sender=Transmission, dispatch_uid="send_mesg")
def send_mesg(sender, instance, **kwargs):
    #log.debug('Hit post save()')
    #log.debug('DATA %s', json.dumps(instance.as_dict()))
    #log.debug('DATA %s', json.dumps(instance.as_dict()))
    tg = TalkGroup.objects.get(pk=instance.talkgroup_info.pk)
    groups = tg.scanlist_set.all()
    for g in groups:
        Group('livecall-scan-'+g.slug, ).send({'text': json.dumps(instance.as_dict())})
    Group('livecall-tg-' + tg.slug, ).send({'text': json.dumps(instance.as_dict())})

    #def save(self, *args, **kwargs):
    #    try:
    #        self.talkgroup_info = TalkGroup.objects.get(dec_id=self.talkgroup)
    #    except TalkGroup.DoesNotExist:
    #        new_tg = TalkGroup(dec_id=self.talkgroup, alpha_tag='UNK')
    #        new_tg.save
    #        self.talkgroup_info = TalkGroup.objects.get(dec_id=self.talkgroup)
    #    super(Transmission, self).save(*args, **kwargs)

class TranmissionUnit(models.Model):
    transmission = models.ForeignKey(Transmission, on_delete=models.CASCADE)
    unit = models.ForeignKey(Unit, on_delete=models.CASCADE)
    order = models.IntegerField()

    class Meta:
        ordering = ["order"]

    def __str__(self):
        return '{} on {}'.format(self.unit,self.transmission)

class ScanList(models.Model):
    created_by = models.ForeignKey(settings.AUTH_USER_MODEL)
    public = models.BooleanField(default=False)
    name = models.CharField(max_length=30, unique=True)
    description = models.CharField(max_length=100)
    talkgroups = models.ManyToManyField(TalkGroup)
    slug = models.SlugField(null=True, blank=True)

    class Meta:
        ordering = ["name"]

    def __str__(self):
        return '{}'.format(self.name)

    def save(self, *args, **kwargs):
        self.slug = slugify(self.name)
        super(ScanList, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return '/scan/{}/'.format(self.slug)

class MenuList(models.Model):
    order = models.IntegerField(default=1)

    class Meta:
        abstract = True
        ordering = ["order", "name"]

    def __str__(self):
        return '{}'.format(self.name)

    @property
    def scan_name(self):
        return self.name

    @property
    def scan_description(self):
        return self.name.description

    @property
    def scan_slug(self):
        return self.name.slug

class MenuScanList(MenuList):
    name = models.ForeignKey(ScanList)

    @property
    def scan_name(self):
        return self.name.name

    @property
    def scan_description(self):
        return self.name.description

class MenuTalkGroupList(MenuList):
    name = models.ForeignKey(TalkGroupWithSystem)

    @property
    def tg_name(self):
        return self.name.alpha_tag

    @property
    def tg_slug(self):
        return self.name.slug

class TalkGroupAccess(models.Model):
    name = models.CharField(max_length=30, unique=True)
    talkgroups = models.ManyToManyField(TalkGroupWithSystem)
    default_group = models.BooleanField(default=False, help_text='new users get this group by default')
    default_new_talkgroups = models.BooleanField(default=False, help_text='new talkgroups are added to this group')

    class Meta:
        verbose_name_plural = 'talk group access'

    def __str__(self):
        return '{}'.format(self.name)

class Plan(models.Model):
    DEFAULT_PK = 1 # This is added via a migration
    name = models.CharField(max_length=30, unique=True)
    history = models.IntegerField(default=15, help_text='visible history in minutes')

    def __str__(self):
        return '{}'.format(self.name)

class Profile(models.Model):
    user = models.OneToOneField(User, on_delete=models.CASCADE)
    plan = models.ForeignKey(Plan, default=Plan.DEFAULT_PK)
    talkgroup_access = models.ManyToManyField(TalkGroupAccess, blank=True)

class WebHtml(models.Model):
    name = models.CharField(max_length=30, unique=True)
    bodytext = models.TextField()

    def __str__(self):
        return self.name

class StripePlanMatrix(models.Model):
    name = models.CharField(max_length=30, unique=True)
    stripe_plan = models.ForeignKey(pinax_Plan)
    radio_plan = models.ForeignKey(Plan)
    active = models.BooleanField(default=True)
    order = models.IntegerField(default=99)

    class Meta:
        ordering = ["order"]

    def __str__(self):
       return self.name

    def stripe_amount(self):
       return int(self.stripe_plan.amount * 100)

    def history_days(self):
       return int(self.radio_plan.history / 1440)

class SiteOption(models.Model):
    name = models.CharField(max_length=30, unique=True)
    value = models.CharField(max_length=60)
    javascript_visible = models.BooleanField(default=False)
    template_visible = models.BooleanField(default=False)
    description = models.CharField(max_length=100, null=True, blank=True)

    def __str__(self):
        return self.name

    def value_boolean_or_string(self):
        if self.value.lower() == 'true':
            return True
        elif self.value.lower() == 'false':
            return False
        else:
            return self.value

def create_profile(sender, **kwargs):
    user = kwargs["instance"]
    if kwargs["created"]:
        default_plan = Plan.objects.get(pk=Plan.DEFAULT_PK)
        up = Profile(user=user, plan=default_plan)
        up.save()
        for tg in TalkGroupAccess.objects.filter(default_group=True):
            up.talkgroup_access.add(tg)
        try:
            new_user_email = SiteOption.objects.get(name='SEND_ADMIN_EMAIL_ON_NEW_USER')
            if new_user_email.value_boolean_or_string() == True:
                send_mail(
                      'New ScanOC User {}'.format(user.username),
                      'New User {} {} Username {} Email {} just registered'.format(user.first_name, user.last_name, user.username, user.email),
                      settings.SERVER_EMAIL,
                      settings.ADMINS,
                      fail_silently=False,
                     )
        except SiteOption.DoesNotExist:
            pass

post_save.connect(create_profile, sender=User)
<commit_msg>Fix To: line in new reg email<commit_after>import json
import logging
import uuid
import urllib.parse

from django.db import models
from django.utils import timezone
from django.utils.text import slugify
from django.conf import settings
from channels import Group
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.contrib.auth.models import User
from django.core.mail import send_mail

import radio.choices as choice

from pinax.stripe.models import Plan as pinax_Plan

log = logging.getLogger(__name__)

class Agency(models.Model):
    name = models.CharField(max_length=100)
    short = models.CharField(max_length=5, unique=True)

    def __str__(self):
        return self.name

    def get_short(self):
        return self.short

class System(models.Model):
    name = models.CharField(max_length=100)
    system_id = models.CharField(max_length=20, blank=True, null=True)

    def __str__(self):
        return self.name

class Source(models.Model):
    description = models.CharField(max_length=100)

    class Meta:
        ordering = ["pk"]

    def __str__(self):
        return self.description

class Unit(models.Model):
    dec_id = models.IntegerField()
    description = models.CharField(max_length=100, blank=True, null=True)
    agency = models.ForeignKey(Agency, default=settings.RADIO_DEFAULT_UNIT_AGENCY)
    #agency = models.ForeignKey(Agency, default=2)
    type = models.CharField(max_length=1, choices=choice.RADIO_TYPE_CHOICES, default=choice.RADIO_TYPE_MOBILE)
    number = models.IntegerField(default=1)
    system = models.ForeignKey(System, default=0)
    slug = models.SlugField(null=True, blank=True)

    class Meta:
        unique_together = ('dec_id', 'system',)

    def __str__(self):
        if(self.description):
            return self.description
        else:
            return str(self.dec_id)

    def save(self, *args, **kwargs):
        if self.description:
            self.slug = slugify(self.description)
        super(Unit, self).save(*args, **kwargs)

class RepeaterSite(models.Model):
    name = models.CharField(max_length=100, unique=True)
    description = models.CharField(max_length=500, blank=True, null=True)

    def __str__(self):
        return self.name

class Service(models.Model):
    name = models.CharField(max_length=100, unique=True)
    description = models.CharField(max_length=500, blank=True, null=True)

    def __str__(self):
        return self.name

class TalkGroup(models.Model):
    dec_id = models.IntegerField()
    alpha_tag = models.CharField(max_length=30)
    common_name = models.CharField(max_length=10, blank=True, null=True)
    description = models.CharField(max_length=100, blank=True, null=True)
    slug = models.SlugField(null=True)
    public = models.BooleanField(default=True)
    comments = models.CharField(max_length=100, blank=True, null=True)
    system = models.ForeignKey(System, default=0)
    mode = models.CharField(max_length=1, choices=choice.TG_MODE_CHOICES, default=choice.TG_MODE_DIGITAL, help_text='mode used by trunk-recorder')
    priority = models.IntegerField(default=3, help_text='record priority used by trunk-recorder')
    _home_site = models.ForeignKey(RepeaterSite, blank=True, null=True)
    _service_type = models.ForeignKey(Service, blank=True, null=True)

    class Meta:
        ordering = ["alpha_tag"]
        unique_together = ('dec_id', 'system',)

    def __str__(self):
        return self.alpha_tag

    def save(self, *args, **kwargs):
        self.slug = slugify(self.alpha_tag)
        super(TalkGroup, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return '/tg/{}/'.format(self.slug)

    @property
    def home_site(self):
        return self._home_site

    @home_site.setter
    def home_site(self, value):
        hs, created = RepeaterSite.objects.get_or_create(name=value)
        self._home_site = hs

    @property
    def service_type(self):
        return self._service_type

    @service_type.setter
    def service_type(self, value):
        st, created = Service.objects.get_or_create(name=value)
        self._service_type = st

def AddToDefaultAccessGroup(sender, instance, created, **kwargs):
    if created:
        new_tg = TalkGroupWithSystem.objects.get(pk=instance.pk)
        for access_group in TalkGroupAccess.objects.filter(default_new_talkgroups=True):
            access_group.talkgroups.add(new_tg)
            access_group.save()

post_save.connect(AddToDefaultAccessGroup, sender=TalkGroup)

class TalkGroupWithSystem(TalkGroup):
    class Meta:
        proxy = True

    def __str__(self):
        return "{} ({})".format(self.alpha_tag, self.system)

class Transmission(models.Model):
    slug = models.UUIDField(db_index=True, default=uuid.uuid4, editable=False)
    start_datetime = models.DateTimeField(db_index=True)
    audio_file = models.FileField()
    audio_file_url_path = models.CharField(max_length=100, default='/')
    talkgroup = models.IntegerField()
    talkgroup_info = models.ForeignKey(TalkGroup)
    freq = models.IntegerField()
    emergency = models.BooleanField(default=False)
    units = models.ManyToManyField(Unit, through='TranmissionUnit')
    play_length = models.FloatField(default=0.0)
    source = models.ForeignKey(Source, default=0)
    system = models.ForeignKey(System, default=0)

    def __str__(self):
        return '{} {}'.format(self.talkgroup, self.start_datetime)

    def get_absolute_url(self):
        return '/audio/{}/'.format(self.slug)

    @property
    def local_start_datetime(self):
        return timezone.localtime(self.start_datetime).strftime('%H:%M:%S %m/%d/%Y')

    def as_dict(self):
        return {'start_datetime': str(self.start_datetime), 'audio_file': str(self.audio_file), 'talkgroup_desc': self.talkgroup_info.alpha_tag}

    def print_play_length(self):
        m, s = divmod(int(self.play_length), 60)
        return '{:02d}:{:02d}'.format(m,s)

    def freq_mhz(self):
        return '{0:07.3f}'.format(self.freq / 1000000)

    def tg_name(self):
        """Returns TalkGroup name used for page title
           First tries TalkGroup.common_name
           then TalkGroup.alpha_tag
        """
        if self.talkgroup_info.common_name:
            return self.talkgroup_info.common_name
        else:
            return self.talkgroup_info.alpha_tag

    @property
    def audio_url(self):
        base_path = settings.AUDIO_URL_BASE
        return urllib.parse.urljoin(base_path, self.audio_file_url_path.lstrip('/'))

    class Meta:
        ordering = ["-pk"]

@receiver(post_save, sender=Transmission, dispatch_uid="send_mesg")
def send_mesg(sender, instance, **kwargs):
    #log.debug('Hit post save()')
    #log.debug('DATA %s', json.dumps(instance.as_dict()))
    #log.debug('DATA %s', json.dumps(instance.as_dict()))
    tg = TalkGroup.objects.get(pk=instance.talkgroup_info.pk)
    groups = tg.scanlist_set.all()
    for g in groups:
        Group('livecall-scan-'+g.slug, ).send({'text': json.dumps(instance.as_dict())})
    Group('livecall-tg-' + tg.slug, ).send({'text': json.dumps(instance.as_dict())})

    #def save(self, *args, **kwargs):
    #    try:
    #        self.talkgroup_info = TalkGroup.objects.get(dec_id=self.talkgroup)
    #    except TalkGroup.DoesNotExist:
    #        new_tg = TalkGroup(dec_id=self.talkgroup, alpha_tag='UNK')
    #        new_tg.save
    #        self.talkgroup_info = TalkGroup.objects.get(dec_id=self.talkgroup)
    #    super(Transmission, self).save(*args, **kwargs)

class TranmissionUnit(models.Model):
    transmission = models.ForeignKey(Transmission, on_delete=models.CASCADE)
    unit = models.ForeignKey(Unit, on_delete=models.CASCADE)
    order = models.IntegerField()

    class Meta:
        ordering = ["order"]

    def __str__(self):
        return '{} on {}'.format(self.unit,self.transmission)

class ScanList(models.Model):
    created_by = models.ForeignKey(settings.AUTH_USER_MODEL)
    public = models.BooleanField(default=False)
    name = models.CharField(max_length=30, unique=True)
    description = models.CharField(max_length=100)
    talkgroups = models.ManyToManyField(TalkGroup)
    slug = models.SlugField(null=True, blank=True)

    class Meta:
        ordering = ["name"]

    def __str__(self):
        return '{}'.format(self.name)

    def save(self, *args, **kwargs):
        self.slug = slugify(self.name)
        super(ScanList, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return '/scan/{}/'.format(self.slug)

class MenuList(models.Model):
    order = models.IntegerField(default=1)

    class Meta:
        abstract = True
        ordering = ["order", "name"]

    def __str__(self):
        return '{}'.format(self.name)

    @property
    def scan_name(self):
        return self.name

    @property
    def scan_description(self):
        return self.name.description

    @property
    def scan_slug(self):
        return self.name.slug

class MenuScanList(MenuList):
    name = models.ForeignKey(ScanList)

    @property
    def scan_name(self):
        return self.name.name

    @property
    def scan_description(self):
        return self.name.description

class MenuTalkGroupList(MenuList):
    name = models.ForeignKey(TalkGroupWithSystem)

    @property
    def tg_name(self):
        return self.name.alpha_tag

    @property
    def tg_slug(self):
        return self.name.slug

class TalkGroupAccess(models.Model):
    name = models.CharField(max_length=30, unique=True)
    talkgroups = models.ManyToManyField(TalkGroupWithSystem)
    default_group = models.BooleanField(default=False, help_text='new users get this group by default')
    default_new_talkgroups = models.BooleanField(default=False, help_text='new talkgroups are added to this group')

    class Meta:
        verbose_name_plural = 'talk group access'

    def __str__(self):
        return '{}'.format(self.name)

class Plan(models.Model):
    DEFAULT_PK = 1 # This is added via a migration
    name = models.CharField(max_length=30, unique=True)
    history = models.IntegerField(default=15, help_text='visible history in minutes')

    def __str__(self):
        return '{}'.format(self.name)

class Profile(models.Model):
    user = models.OneToOneField(User, on_delete=models.CASCADE)
    plan = models.ForeignKey(Plan, default=Plan.DEFAULT_PK)
    talkgroup_access = models.ManyToManyField(TalkGroupAccess, blank=True)

class WebHtml(models.Model):
    name = models.CharField(max_length=30, unique=True)
    bodytext = models.TextField()

    def __str__(self):
        return self.name

class StripePlanMatrix(models.Model):
    name = models.CharField(max_length=30, unique=True)
    stripe_plan = models.ForeignKey(pinax_Plan)
    radio_plan = models.ForeignKey(Plan)
    active = models.BooleanField(default=True)
    order = models.IntegerField(default=99)

    class Meta:
        ordering = ["order"]

    def __str__(self):
       return self.name

    def stripe_amount(self):
       return int(self.stripe_plan.amount * 100)

    def history_days(self):
       return int(self.radio_plan.history / 1440)

class SiteOption(models.Model):
    name = models.CharField(max_length=30, unique=True)
    value = models.CharField(max_length=60)
    javascript_visible = models.BooleanField(default=False)
    template_visible = models.BooleanField(default=False)
    description = models.CharField(max_length=100, null=True, blank=True)

    def __str__(self):
        return self.name

    def value_boolean_or_string(self):
        if self.value.lower() == 'true':
            return True
        elif self.value.lower() == 'false':
            return False
        else:
            return self.value

def create_profile(sender, **kwargs):
    user = kwargs["instance"]
    if kwargs["created"]:
        default_plan = Plan.objects.get(pk=Plan.DEFAULT_PK)
        up = Profile(user=user, plan=default_plan)
        up.save()
        for tg in TalkGroupAccess.objects.filter(default_group=True):
            up.talkgroup_access.add(tg)
        try:
            new_user_email = SiteOption.objects.get(name='SEND_ADMIN_EMAIL_ON_NEW_USER')
            if new_user_email.value_boolean_or_string() == True:
                send_mail(
                      'New ScanOC User {}'.format(user.username),
                      'New User {} {} Username {} Email {} just registered'.format(user.first_name, user.last_name, user.username, user.email),
                      settings.SERVER_EMAIL,
                      [ mail for name, mail in settings.ADMINS],
                      fail_silently=False,
                     )
        except SiteOption.DoesNotExist:
            pass

post_save.connect(create_profile, sender=User)
<|endoftext|>