<commit_before>import sys
import argparse
import tornado.web

from zmq.eventloop import ioloop
ioloop.install()

from crypto2crypto import CryptoTransportLayer
from db_store import Obdb
from market import Market
from ws import WebSocketHandler
import logging
import signal
from threading import Thread
from twisted.internet import reactor
import upnp

class MainHandler(tornado.web.RequestHandler):
    def get(self):
        self.redirect("/html/index.html")

class MarketApplication(tornado.web.Application):
    def __init__(self, market_ip, market_port, market_id=1,
                 bm_user=None, bm_pass=None, bm_port=None, seed_peers=[],
                 seed_mode=0, dev_mode=False, db_path='db/ob.db'):

        db = Obdb(db_path)

        self.transport = CryptoTransportLayer(market_ip,
                                              market_port,
                                              market_id,
                                              db,
                                              bm_user,
                                              bm_pass,
                                              bm_port,
                                              seed_mode,
                                              dev_mode)

        self.market = Market(self.transport, db)

        def post_joined():
            self.transport._dht._refreshNode()
            self.market.republish_contracts()

        peers = seed_peers if seed_mode == 0 else []
        self.transport.join_network(peers)

        Thread(target=reactor.run, args=(False,)).start()

        handlers = [
            (r"/", MainHandler),
            (r"/main", MainHandler),
            (r"/html/(.*)", tornado.web.StaticFileHandler, {'path': './html'}),
            (r"/ws", WebSocketHandler,
                dict(transport=self.transport, market=self.market, db=db))
        ]

        # TODO: Move debug settings to configuration location
        settings = dict(debug=True)
        tornado.web.Application.__init__(self, handlers, **settings)

    def get_transport(self):
        return self.transport

    def setup_upnp_port_mappings(self, http_port, p2p_port):
        upnp.PortMapper.DEBUG = False
        print "Setting up UPnP Port Map Entry..."
        # TODO: Add some setting whether or not to use UPnP
        # if Settings.get(Settings.USE_UPNP_PORT_MAPPINGS):
        self.upnp_mapper = upnp.PortMapper()
        # TODO: Add some setting whether or not to clean all previous port
        # mappings left behind by us
        # if Settings.get(Settings.CLEAN_UPNP_PORT_MAPPINGS_ON_START):
        #    upnp_mapper.cleanMyMappings()

        # for now let's always clean mappings every time.
        self.upnp_mapper.clean_my_mappings()
        result_http_port_mapping = self.upnp_mapper.add_port_mapping(http_port,
                                                                     http_port)
        print ("UPnP HTTP Port Map configuration done (%s -> %s) => %s" %
               (str(http_port), str(http_port), str(result_http_port_mapping)))

        result_tcp_p2p_mapping = self.upnp_mapper.add_port_mapping(p2p_port,
                                                                   p2p_port)
        print ("UPnP TCP P2P Port Map configuration done (%s -> %s) => %s" %
               (str(p2p_port), str(p2p_port), str(result_tcp_p2p_mapping)))

        result_udp_p2p_mapping = self.upnp_mapper.add_port_mapping(p2p_port,
                                                                   p2p_port,
                                                                   'UDP')
        print ("UPnP UDP P2P Port Map configuration done (%s -> %s) => %s" %
               (str(p2p_port), str(p2p_port), str(result_udp_p2p_mapping)))

        return result_http_port_mapping and \
            result_tcp_p2p_mapping and \
            result_udp_p2p_mapping

    def cleanup_upnp_port_mapping(self):
        if self.upnp_mapper is not None:
            print "Cleaning UPnP Port Mapping -> ", \
                self.upnp_mapper.clean_my_mappings()

def start_node(my_market_ip,
               my_market_port,
               log_file,
               market_id,
               bm_user=None,
               bm_pass=None,
               bm_port=None,
               seed_peers=[],
               seed_mode=0,
               dev_mode=False,
               log_level=None,
               database='db/ob.db',
               disable_upnp=False):

    logging.basicConfig(level=int(log_level),
                        format='%(asctime)s - %(name)s -  \
                                %(levelname)s - %(message)s',
                        filename=log_file)

    locallogger = logging.getLogger('[%s] %s' % (market_id, 'root'))

    handler = logging.handlers.RotatingFileHandler(log_file,
                                                   maxBytes=50,
                                                   backupCount=0)
    locallogger.addHandler(handler)

    application = MarketApplication(my_market_ip,
                                    my_market_port,
                                    market_id,
                                    bm_user,
                                    bm_pass,
                                    bm_port,
                                    seed_peers,
                                    seed_mode,
                                    dev_mode,
                                    database)

    error = True
    http_port = 8888
    p2p_port = 12345

    while error and http_port < 8988:
        try:
            application.listen(http_port)
            error = False
        except:
            http_port += 1

    if not disable_upnp:
        application.setup_upnp_port_mappings(http_port, p2p_port)
    else:
        print "Disabling upnp setup"

    locallogger.info("Started OpenBazaar Web App at http://%s:%s" %
                     (my_market_ip, http_port))
    print "Started OpenBazaar Web App at http://%s:%s" % (my_market_ip, http_port)

    # handle shutdown
    def shutdown(x, y):
        locallogger = logging.getLogger('[%s] %s' % (market_id, 'root'))
        locallogger.info("Received TERMINATE, exiting...")

        # application.get_transport().broadcast_goodbye()
        application.cleanup_upnp_port_mapping()
        application.market.p.kill()

        sys.exit(0)
    try:
        signal.signal(signal.SIGTERM, shutdown)
    except ValueError:
        # not the main thread
        pass

    if not tornado.ioloop.IOLoop.instance():
        ioloop.install()
    else:
        tornado.ioloop.IOLoop.instance().start()

# Run this if executed directly
if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("my_market_ip")
    parser.add_argument("-p", "--my_market_port",
                        type=int, default=12345)
    parser.add_argument("-l", "--log_file",
                        default='logs/production.log')
    parser.add_argument("-u", "--market_id",
                        default=1)
    parser.add_argument("-S", "--seed_peers",
                        nargs='*', default=[])
    parser.add_argument("-s", "--seed_mode",
                        default=0)
    parser.add_argument("-d", "--dev_mode",
                        action='store_true')
    parser.add_argument("--database",
                        default='db/ob.db', help="Database filename")
    parser.add_argument("--bmuser",
                        default='username', help="Bitmessage instance user")
    parser.add_argument("--bmpass",
                        default='password', help="Bitmessage instance pass")
    parser.add_argument("--bmport",
                        default='8442', help="Bitmessage instance RPC port")
    parser.add_argument("--log_level",
                        default=10, help="Numeric value for logging level")
    parser.add_argument("--disable_upnp",
                        action='store_true')
    args = parser.parse_args()
    start_node(args.my_market_ip,
               args.my_market_port,
               args.log_file,
               args.market_id,
               args.bmuser,
               args.bmpass,
               args.bmport,
               args.seed_peers,
               args.seed_mode,
               args.dev_mode,
               args.log_level,
               args.database,
               args.disable_upnp)
<commit_msg>stop the IOLoop instead in the meantime. Other things should probably be shutdown here. #225<commit_after>import sys
import argparse
import tornado.web

from zmq.eventloop import ioloop
ioloop.install()

from crypto2crypto import CryptoTransportLayer
from db_store import Obdb
from market import Market
from ws import WebSocketHandler
import logging
import signal
from threading import Thread
from twisted.internet import reactor
import upnp

class MainHandler(tornado.web.RequestHandler):
    def get(self):
        self.redirect("/html/index.html")

class MarketApplication(tornado.web.Application):
    def __init__(self, market_ip, market_port, market_id=1,
                 bm_user=None, bm_pass=None, bm_port=None, seed_peers=[],
                 seed_mode=0, dev_mode=False, db_path='db/ob.db'):

        db = Obdb(db_path)

        self.transport = CryptoTransportLayer(market_ip,
                                              market_port,
                                              market_id,
                                              db,
                                              bm_user,
                                              bm_pass,
                                              bm_port,
                                              seed_mode,
                                              dev_mode)

        self.market = Market(self.transport, db)

        def post_joined():
            self.transport._dht._refreshNode()
            self.market.republish_contracts()

        peers = seed_peers if seed_mode == 0 else []
        self.transport.join_network(peers)

        Thread(target=reactor.run, args=(False,)).start()

        handlers = [
            (r"/", MainHandler),
            (r"/main", MainHandler),
            (r"/html/(.*)", tornado.web.StaticFileHandler, {'path': './html'}),
            (r"/ws", WebSocketHandler,
                dict(transport=self.transport, market=self.market, db=db))
        ]

        # TODO: Move debug settings to configuration location
        settings = dict(debug=True)
        tornado.web.Application.__init__(self, handlers, **settings)

    def get_transport(self):
        return self.transport

    def setup_upnp_port_mappings(self, http_port, p2p_port):
        upnp.PortMapper.DEBUG = False
        print "Setting up UPnP Port Map Entry..."
        # TODO: Add some setting whether or not to use UPnP
        # if Settings.get(Settings.USE_UPNP_PORT_MAPPINGS):
        self.upnp_mapper = upnp.PortMapper()
        # TODO: Add some setting whether or not to clean all previous port
        # mappings left behind by us
        # if Settings.get(Settings.CLEAN_UPNP_PORT_MAPPINGS_ON_START):
        #    upnp_mapper.cleanMyMappings()

        # for now let's always clean mappings every time.
        self.upnp_mapper.clean_my_mappings()
        result_http_port_mapping = self.upnp_mapper.add_port_mapping(http_port,
                                                                     http_port)
        print ("UPnP HTTP Port Map configuration done (%s -> %s) => %s" %
               (str(http_port), str(http_port), str(result_http_port_mapping)))

        result_tcp_p2p_mapping = self.upnp_mapper.add_port_mapping(p2p_port,
                                                                   p2p_port)
        print ("UPnP TCP P2P Port Map configuration done (%s -> %s) => %s" %
               (str(p2p_port), str(p2p_port), str(result_tcp_p2p_mapping)))

        result_udp_p2p_mapping = self.upnp_mapper.add_port_mapping(p2p_port,
                                                                   p2p_port,
                                                                   'UDP')
        print ("UPnP UDP P2P Port Map configuration done (%s -> %s) => %s" %
               (str(p2p_port), str(p2p_port), str(result_udp_p2p_mapping)))

        return result_http_port_mapping and \
            result_tcp_p2p_mapping and \
            result_udp_p2p_mapping

    def cleanup_upnp_port_mapping(self):
        if self.upnp_mapper is not None:
            print "Cleaning UPnP Port Mapping -> ", \
                self.upnp_mapper.clean_my_mappings()

def start_node(my_market_ip,
               my_market_port,
               log_file,
               market_id,
               bm_user=None,
               bm_pass=None,
               bm_port=None,
               seed_peers=[],
               seed_mode=0,
               dev_mode=False,
               log_level=None,
               database='db/ob.db',
               disable_upnp=False):

    logging.basicConfig(level=int(log_level),
                        format='%(asctime)s - %(name)s -  \
                                %(levelname)s - %(message)s',
                        filename=log_file)

    locallogger = logging.getLogger('[%s] %s' % (market_id, 'root'))

    handler = logging.handlers.RotatingFileHandler(log_file,
                                                   maxBytes=50,
                                                   backupCount=0)
    locallogger.addHandler(handler)

    application = MarketApplication(my_market_ip,
                                    my_market_port,
                                    market_id,
                                    bm_user,
                                    bm_pass,
                                    bm_port,
                                    seed_peers,
                                    seed_mode,
                                    dev_mode,
                                    database)

    error = True
    http_port = 8888
    p2p_port = 12345

    while error and http_port < 8988:
        try:
            application.listen(http_port)
            error = False
        except:
            http_port += 1

    if not disable_upnp:
        application.setup_upnp_port_mappings(http_port, p2p_port)
    else:
        print "Disabling upnp setup"

    locallogger.info("Started OpenBazaar Web App at http://%s:%s" %
                     (my_market_ip, http_port))
    print "Started OpenBazaar Web App at http://%s:%s" % (my_market_ip, http_port)

    # handle shutdown
    def shutdown(x, y):
        locallogger = logging.getLogger('[%s] %s' % (market_id, 'root'))
        locallogger.info("Received TERMINATE, exiting...")

        # application.get_transport().broadcast_goodbye()
        application.cleanup_upnp_port_mapping()
        tornado.ioloop.IOLoop.instance().stop()

        #TODO:
        # we should implement the shutdown of the dht connections, db connection, bitmessage connection
        # maybe this was meant to do all that but nobody ever got around it.
        #application.market.p.kill()
        sys.exit(0)
    try:
        signal.signal(signal.SIGTERM, shutdown)
    except ValueError:
        # not the main thread
        pass

    if not tornado.ioloop.IOLoop.instance():
        ioloop.install()
    else:
        tornado.ioloop.IOLoop.instance().start()

# Run this if executed directly
if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("my_market_ip")
    parser.add_argument("-p", "--my_market_port",
                        type=int, default=12345)
    parser.add_argument("-l", "--log_file",
                        default='logs/production.log')
    parser.add_argument("-u", "--market_id",
                        default=1)
    parser.add_argument("-S", "--seed_peers",
                        nargs='*', default=[])
    parser.add_argument("-s", "--seed_mode",
                        default=0)
    parser.add_argument("-d", "--dev_mode",
                        action='store_true')
    parser.add_argument("--database",
                        default='db/ob.db', help="Database filename")
    parser.add_argument("--bmuser",
                        default='username', help="Bitmessage instance user")
    parser.add_argument("--bmpass",
                        default='password', help="Bitmessage instance pass")
    parser.add_argument("--bmport",
                        default='8442', help="Bitmessage instance RPC port")
    parser.add_argument("--log_level",
                        default=10, help="Numeric value for logging level")
    parser.add_argument("--disable_upnp",
                        action='store_true')
    args = parser.parse_args()
    start_node(args.my_market_ip,
               args.my_market_port,
               args.log_file,
               args.market_id,
               args.bmuser,
               args.bmpass,
               args.bmport,
               args.seed_peers,
               args.seed_mode,
               args.dev_mode,
               args.log_level,
               args.database,
               args.disable_upnp)
<|endoftext|>


<commit_before>from django.db import models
from django.conf import settings
from django.core import serializers
import tempfile
import os

verbose = False
RASTDIR = '/Users/perry/src/marinemap/lingcod/raster_stats/test_data'
RASTER_TYPES = (
                ("continuous", "continuous"),
                ("categorical", "catgorical"),
               )
try:
    STARSPAN_BIN = settings.STARSPAN_BIN
except:
    STARSPAN_BIN = 'starspan'

class RasterDataset(models.Model):
    name = models.CharField(max_length=30, unique=True)
    filepath = models.FilePathField(path=RASTDIR, recursive=True)
    type = models.CharField(max_length=30, choices=RASTER_TYPES)

class ZonalStatsCache(models.Model):
    geom_hash = models.CharField(max_length=255)
    raster = models.ForeignKey('RasterDataset')
    avg = models.FloatField(null=True, blank=True)
    min = models.FloatField(null=True, blank=True)
    max = models.FloatField(null=True, blank=True)
    mode = models.FloatField(null=True, blank=True)
    median = models.FloatField(null=True, blank=True)
    stdev = models.FloatField(null=True, blank=True)
    nulls = models.FloatField(null=True, blank=True)
    pixels = models.FloatField(null=True, blank=True)
    date_modified = models.DateTimeField(auto_now=True)

    @property
    def json(self):
        return serializers.serialize("json", self)

    def __unicode__(self):
        return unicode("Zonal Stats - avg:%s , min:%s, max:%s" % (self.avg, self.min, self.max))

    class Meta:
        unique_together = ('geom_hash', 'raster')

def geom_to_file(geom, filepath):
    json = """{
"type": "FeatureCollection",
"features": [
{ "type": "Feature", "properties": { "id": 1 }, "geometry": %s }
]
}""" % geom.json
    fh = open(filepath,'w')
    fh.write(json)
    fh.close()
    assert os.path.exists(filepath)

def run_starspan_zonal(geom, rasterds, write_cache=False):
    """
    Consider this a 'private' method .. dont call directly, use zonal_stats() instead
    Runs starspan and returns a ZonalStatsCache object
    If not write_cache, just return an unsaved object
    """
    # Create tempdir and cd in
    tmpdir = tempfile.gettempdir()
    os.chdir(tmpdir)

    # Output geom to temp dataset
    out_json = os.path.join(tmpdir, 'geom.json')
    geom_to_file(geom, out_json)

    # Run starspan
    out_csv = os.path.join(tmpdir, 'output_stats.csv')
    if os.path.exists(out_csv):
        os.remove(out_csv)
    cmd = '%s --vector %s --where "id=1" --out-prefix %s/output --out-type table --summary-suffix _stats.csv --raster %s --stats avg mode median min max sum stdev nulls ' % (STARSPAN_BIN,out_json,tmpdir, rasterds.filepath)
    if verbose: print cmd
    starspan_out = os.popen(cmd).read()
    if verbose: print starspan_out

    # Parse output
    try:
        res = open(out_csv,'r').readlines()
    except IOError:
        print "Starspan failed to create output csv properly"
        raise Exception
    if verbose: print res

    # Create zonal model
    hash = geom.wkt.__hash__()
    zonal = ZonalStatsCache(raster=rasterds, geom_hash=hash)

    # Make sure we have valid results output by starspan
    if len(res) == 2 and "Intersecting features: 0" not in starspan_out:
        headers = [x.strip() for x in res[0].split(',')]
        vals = [x.strip() for x in res[1].split(',')]
        assert len(headers) == len(vals)

        # loop and populate model
        for i in range(len(headers)):
            if "_Band1" in headers[i]:
                stat_type = headers[i].replace("_Band1",'')
                zonal.__dict__[stat_type] = float(vals[i])
            elif headers[i] == 'numPixels':
                zonal.pixels = float(vals[i])

    # return zonal object (caching it if needed)
    if write_cache:
        zonal.save()
    return zonal

def clear_cache():
    objs = ZonalStatsCache.objects.all()
    if verbose: print "Clearing %s objects from cache" % len(objs)
    objs.delete()

def zonal_stats(geom, rasterds, write_cache=True, read_cache=True):
    """
    Given a GEOSGeometry and a RasterDataset,
    compute the zonal stats and return json like
     { 'raster': 'elevation', 'stats': {'sum': 10234.2, 'mean': 12.4}}
    result can be stored in cache (write_cache)
     and cache value is returned if read_cache
    """
    if not geom.valid:
        return None

    hash = geom.wkt.__hash__()

    cached = None
    if read_cache:
        try:
            cached = ZonalStatsCache.objects.get(geom_hash=hash, raster=rasterds)
        except ZonalStatsCache.DoesNotExist:
            cached = None
    else:
        write_cache = False #If we're not reading the cache, we're not going to write to it either

    if cached:
        result = cached
        result.from_cache = True
    else:
        result = run_starspan_zonal(geom, rasterds, write_cache=write_cache)
        result.from_cache = False

    return result
<commit_msg>Make raster dir the test_data directory under lingcod; this will def change in the future<commit_after>from django.db import models
from django.conf import settings
from django.core import serializers
import tempfile
import os

verbose = False
RASTDIR = os.path.join(os.path.dirname(__file__), 'test_data')
RASTER_TYPES = (
                ("continuous", "continuous"),
                ("categorical", "catgorical"),
               )
try:
    STARSPAN_BIN = settings.STARSPAN_BIN
except:
    STARSPAN_BIN = 'starspan'

class RasterDataset(models.Model):
    name = models.CharField(max_length=30, unique=True)
    filepath = models.FilePathField(path=RASTDIR, recursive=True)
    type = models.CharField(max_length=30, choices=RASTER_TYPES)

class ZonalStatsCache(models.Model):
    geom_hash = models.CharField(max_length=255)
    raster = models.ForeignKey('RasterDataset')
    avg = models.FloatField(null=True, blank=True)
    min = models.FloatField(null=True, blank=True)
    max = models.FloatField(null=True, blank=True)
    mode = models.FloatField(null=True, blank=True)
    median = models.FloatField(null=True, blank=True)
    stdev = models.FloatField(null=True, blank=True)
    nulls = models.FloatField(null=True, blank=True)
    pixels = models.FloatField(null=True, blank=True)
    date_modified = models.DateTimeField(auto_now=True)

    @property
    def json(self):
        return serializers.serialize("json", self)

    def __unicode__(self):
        return unicode("Zonal Stats - avg:%s , min:%s, max:%s" % (self.avg, self.min, self.max))

    class Meta:
        unique_together = ('geom_hash', 'raster')

def geom_to_file(geom, filepath):
    json = """{
"type": "FeatureCollection",
"features": [
{ "type": "Feature", "properties": { "id": 1 }, "geometry": %s }
]
}""" % geom.json
    fh = open(filepath,'w')
    fh.write(json)
    fh.close()
    assert os.path.exists(filepath)

def run_starspan_zonal(geom, rasterds, write_cache=False):
    """
    Consider this a 'private' method .. dont call directly, use zonal_stats() instead
    Runs starspan and returns a ZonalStatsCache object
    If not write_cache, just return an unsaved object
    """
    # Create tempdir and cd in
    tmpdir = tempfile.gettempdir()
    os.chdir(tmpdir)

    # Output geom to temp dataset
    out_json = os.path.join(tmpdir, 'geom.json')
    geom_to_file(geom, out_json)

    # Run starspan
    out_csv = os.path.join(tmpdir, 'output_stats.csv')
    if os.path.exists(out_csv):
        os.remove(out_csv)
    cmd = '%s --vector %s --where "id=1" --out-prefix %s/output --out-type table --summary-suffix _stats.csv --raster %s --stats avg mode median min max sum stdev nulls ' % (STARSPAN_BIN,out_json,tmpdir, rasterds.filepath)
    if verbose: print cmd
    starspan_out = os.popen(cmd).read()
    if verbose: print starspan_out

    # Parse output
    try:
        res = open(out_csv,'r').readlines()
    except IOError:
        print "Starspan failed to create output csv properly"
        raise Exception
    if verbose: print res

    # Create zonal model
    hash = geom.wkt.__hash__()
    zonal = ZonalStatsCache(raster=rasterds, geom_hash=hash)

    # Make sure we have valid results output by starspan
    if len(res) == 2 and "Intersecting features: 0" not in starspan_out:
        headers = [x.strip() for x in res[0].split(',')]
        vals = [x.strip() for x in res[1].split(',')]
        assert len(headers) == len(vals)

        # loop and populate model
        for i in range(len(headers)):
            if "_Band1" in headers[i]:
                stat_type = headers[i].replace("_Band1",'')
                zonal.__dict__[stat_type] = float(vals[i])
            elif headers[i] == 'numPixels':
                zonal.pixels = float(vals[i])

    # return zonal object (caching it if needed)
    if write_cache:
        zonal.save()
    return zonal

def clear_cache():
    objs = ZonalStatsCache.objects.all()
    if verbose: print "Clearing %s objects from cache" % len(objs)
    objs.delete()

def zonal_stats(geom, rasterds, write_cache=True, read_cache=True):
    """
    Given a GEOSGeometry and a RasterDataset,
    compute the zonal stats and return json like
     { 'raster': 'elevation', 'stats': {'sum': 10234.2, 'mean': 12.4}}
    result can be stored in cache (write_cache)
     and cache value is returned if read_cache
    """
    if not geom.valid:
        return None

    hash = geom.wkt.__hash__()

    cached = None
    if read_cache:
        try:
            cached = ZonalStatsCache.objects.get(geom_hash=hash, raster=rasterds)
        except ZonalStatsCache.DoesNotExist:
            cached = None
    else:
        write_cache = False #If we're not reading the cache, we're not going to write to it either

    if cached:
        result = cached
        result.from_cache = True
    else:
        result = run_starspan_zonal(geom, rasterds, write_cache=write_cache)
        result.from_cache = False

    return result
<|endoftext|>


<commit_before># -*- coding: utf-8 -*_
from django.db import models

from ..base import ModelTranslationBase
from ..mixins import ModelMixin, ManagerMixin

class FooManager(ManagerMixin, models.Manager):
    pass

class BarManager(ManagerMixin, models.Manager):
    pass

class FooModel(ModelMixin, models.Model):
    title = models.CharField(max_length=255, null=True, blank=True)
    excerpt = models.TextField(blank=True)
    body = models.TextField(blank=True)
    created_at = models.DateTimeField(auto_now_add=True)
    objects = FooManager()

class FooTranslation(ModelTranslationBase):
    model = FooModel
    identifier = 'foo'
    fields = ('title', )

class BarModel(ModelMixin, models.Model):
    title = models.CharField(max_length=255, null=True, blank=True)
    objects = BarManager()

class BarTranslation(ModelTranslationBase):
    model = BarModel
    identifier = 'bar'
    fields = ('title', )

class BadTranslation(object):
    pass

class BadModel(object):
    pass
<commit_msg>Tests: add new translatable fields.<commit_after># -*- coding: utf-8 -*_
from django.db import models

from ..base import ModelTranslationBase
from ..mixins import ModelMixin, ManagerMixin

class FooManager(ManagerMixin, models.Manager):
    pass

class BarManager(ManagerMixin, models.Manager):
    pass

class FooModel(ModelMixin, models.Model):
    title = models.CharField(max_length=255, null=True, blank=True)
    excerpt = models.TextField(blank=True)
    body = models.TextField(blank=True)
    created_at = models.DateTimeField(auto_now_add=True)
    objects = FooManager()

class FooTranslation(ModelTranslationBase):
    model = FooModel
    identifier = 'foo'
    fields = ('title', 'excerpt', 'body')

class BarModel(ModelMixin, models.Model):
    title = models.CharField(max_length=255, null=True, blank=True)
    objects = BarManager()

class BarTranslation(ModelTranslationBase):
    model = BarModel
    identifier = 'bar'
    fields = ('title', )

class BadTranslation(object):
    pass

class BadModel(object):
    pass
<|endoftext|>


<commit_before># Lint as: python2, python3
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for base_layer."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import lingvo.compat as tf
from lingvo.core import base_layer
from lingvo.core import hyperparams
from lingvo.core import py_utils
from lingvo.core import test_utils

class AddingAccumulator(base_layer.Accumulator):

  def DefaultValue(self):
    return tf.convert_to_tensor(0.0)

  def Update(self, new_value):
    self.SetValue(self.GetValue() + new_value)

def EvalAndFlatten(nmap):
  return nmap.Transform(lambda x: x.eval()).FlattenItems()

class TestLayer(base_layer.BaseLayer):

  @base_layer.initializer
  def __init__(self, params):
    super(TestLayer, self).__init__(params)
    p = self.params
    with tf.variable_scope(p.name):
      self.CreateVariable(
          'w',
          py_utils.WeightParams(
              shape=[4, 4],
              dtype=p.dtype,
              init=p.params_init,
              collections=[self.__class__.__name__ + '_vars']))
      self.CreateVariable(
          'b',
          py_utils.WeightParams(
              shape=[4],
              dtype=p.dtype,
              init=py_utils.WeightInit.Constant(),
              collections=[
                  self.__class__.__name__ + '_vars',
                  py_utils.SKIP_LP_REGULARIZATION
              ]))

class BaseLayerTest(test_utils.TestCase):

  def testCopyBaseParams(self):
    # CopyBaseParams should only overwrite vn setting when target use
    # default vn config.
    layer_base_p = base_layer.BaseLayer.Params()
    from_param = layer_base_p.Copy()
    to_param = layer_base_p.Copy()
    from_param.vn.global_vn = True
    from_param.random_seed = 1234
    from_param.skip_lp_regularization = True
    # Target use default, overwrite.
    base_layer.BaseLayer.CopyBaseParams(from_param, to_param)
    self.assertTrue(to_param.vn.global_vn)
    self.assertEqual(1234, to_param.random_seed)
    self.assertTrue(to_param.skip_lp_regularization)
    to_param = layer_base_p.Copy()
    to_param.vn.per_step_vn = True
    to_param.random_seed = 4321
    to_param.skip_lp_regularization = False
    # Target does not use default, should not overwrite.
    base_layer.BaseLayer.CopyBaseParams(from_param, to_param)
    self.assertTrue(to_param.vn.per_step_vn)
    self.assertFalse(to_param.vn.global_vn)
    self.assertEqual(4321, to_param.random_seed)
    self.assertFalse(to_param.skip_lp_regularization)

  def testCreateChildren(self):
    layer_p = base_layer.BaseLayer.Params()
    layer_p.name = 'test'
    layer = layer_p.Instantiate()
    layer.CreateChildren('a', [layer_p, [layer_p, layer_p], layer_p])
    self.assertEqual(len(layer.a), 3)
    self.assertEqual(len(layer.a[1]), 2)
    self.assertEqual(len(layer.vars.a), 3)
    self.assertEqual(len(layer.vars.a[1]), 2)
    self.assertEqual(len(layer.theta.a), 3)
    self.assertEqual(len(layer.theta.a[1]), 2)

  def testCreateVariable(self):
    layer_p = TestLayer.Params().Set(name='test')
    layer = layer_p.Instantiate()
    self.assertEqual('test/w/var:0', layer.vars.w.name)
    self.assertEqual('test/b/var:0', layer.vars.b.name)
    self.assertNotIn(layer.vars.w,
                     tf.get_collection(py_utils.SKIP_LP_REGULARIZATION))
    # 'b' always skips Lp regularization.
    self.assertIn(layer.vars.b,
                  tf.get_collection(py_utils.SKIP_LP_REGULARIZATION))

  def testCreateVariableSkipLpRegularization(self):
    layer_p = TestLayer.Params().Set(name='test', skip_lp_regularization=True)
    layer = layer_p.Instantiate()
    self.assertIn(layer.vars.w,
                  tf.get_collection(py_utils.SKIP_LP_REGULARIZATION))
    self.assertIn(layer.vars.b,
                  tf.get_collection(py_utils.SKIP_LP_REGULARIZATION))

  def testGetDescendant(self):
    q = base_layer.BaseLayer.Params()
    q.name = 'test'
    l = q.Instantiate()
    p = base_layer.BaseLayer.Params()
    l.CreateChild('a', p)
    l.CreateChild('b', p)
    l.a.CreateChild('c', p)
    l.a.c.CreateChild('d', p)
    l.b.CreateChild('e', p)
    self.assertEqual(l, l.GetDescendant(''))
    self.assertEqual(l.a, l.GetDescendant('a'))
    self.assertEqual(l.b, l.GetDescendant('b'))
    self.assertEqual(l.a.c, l.GetDescendant('a.c'))
    self.assertEqual(l.a.c.d, l.GetDescendant('a.c.d'))
    self.assertEqual(l.b.e, l.GetDescendant('b.e'))

  def testCreateAccumulator(self):
    layer_p = base_layer.BaseLayer.Params()
    layer_p.name = 'test'
    layer = layer_p.Instantiate()
    layer.CreateChild('child', layer_p)

    # First accumulator should succeed.
    acc1 = AddingAccumulator()
    layer.RegisterAccumulator('acc1', acc1)

    # Name of existing child should fail.
    with self.assertRaises(AssertionError):
      layer.RegisterAccumulator('child', AddingAccumulator())

    # Duplicate should fail.
    with self.assertRaises(AssertionError):
      layer.RegisterAccumulator('acc1', AddingAccumulator())

    # Child with the same name should fail.
    with self.assertRaises(AssertionError):
      layer.CreateChild('acc1', layer_p)

    self.assertEqual(acc1, layer.accumulators.acc1)

    # Get of not created accumulator should fail.
    with self.assertRaises(AttributeError):
      layer.accumulators.notexist

  def testGetUpdateAccumulator(self):
    with self.session():
      layer_p = base_layer.BaseLayer.Params()
      layer_p.name = 'test'
      layer = layer_p.Instantiate()

      layer.RegisterAccumulator('acc1', AddingAccumulator())

      # Initial value.
      self.assertEqual(0.0, layer.accumulators.acc1.GetValue().eval())

      # Update/merge.
      layer.accumulators.acc1.Update(1.0)
      layer.accumulators.acc1.Update(1.0)
      self.assertEqual(2.0, layer.accumulators.acc1.GetValue().eval())

      # Reset.
      layer.accumulators.Transform(lambda acc: acc.Reset())
      self.assertEqual(0.0, layer.accumulators.acc1.GetValue().eval())

  def testAccumulatorDisableEnable(self):
    with self.session():
      layer_p = base_layer.BaseLayer.Params()
      layer_p.name = 'test'
      layer = layer_p.Instantiate()

      layer.RegisterAccumulator('acc1', AddingAccumulator())
      layer.accumulators.acc1.Update(1.0)

      # Disable should force value to 0 and reject updates.
      layer.accumulators.Transform(lambda acc: acc.Disable())
      self.assertEqual(0.0, layer.accumulators.acc1.GetValue().eval())
      layer.accumulators.acc1.Update(3.0)
      self.assertEqual(0.0, layer.accumulators.acc1.GetValue().eval())
      layer.accumulators.Transform(lambda acc: acc.Enable())

      # Should restore.
      self.assertEqual(1.0, layer.accumulators.acc1.GetValue().eval())

  def testGetSetAccumulatorValues(self):
    with self.session():
      layer_p = base_layer.BaseLayer.Params()
      layer_p.name = 'test'
      layer1 = layer_p.Instantiate()
      layer1.CreateChild('layer1a', layer_p)
      layer1.CreateChild('layer1b', layer_p)
      layer1.layer1b.CreateChild('layer1b1', layer_p)

      # Create nested accumulators:
      #   acc1: layer1
      #   acc2: layer1.layer1a
      #   acc3: layer1.layer1b.layer1b1
      layer1.RegisterAccumulator('acc1', AddingAccumulator())
      layer1.layer1a.RegisterAccumulator('acc2', AddingAccumulator())
      layer1.layer1b.layer1b1.RegisterAccumulator('acc3', AddingAccumulator())

      # Pack with initial values.
      initial_pack = layer1.GetAccumulatorValues()
      initial_pack_eval = EvalAndFlatten(initial_pack)
      print('Initial values pack =', initial_pack_eval)
      self.assertEqual(initial_pack_eval, [('acc1', 0.0), ('layer1a.acc2', 0.0),
                                           ('layer1b.layer1b1.acc3', 0.0)])

      # Update to a new known state.
      layer1.accumulators.acc1.Update(1.0)
      layer1.layer1a.accumulators.acc2.Update(2.0)
      layer1.layer1b.layer1b1.accumulators.acc3.Update(3.0)
      updated_pack = layer1.GetAccumulatorValues()
      updated_pack_eval = EvalAndFlatten(updated_pack)
      print('Updated values pack =', updated_pack_eval)
      self.assertEqual(updated_pack_eval, [('acc1', 1.0), ('layer1a.acc2', 2.0),
                                           ('layer1b.layer1b1.acc3', 3.0)])

      # Save and reset.
      saved_pack = layer1.GetAccumulatorValues()
      layer1.accumulators.Transform(lambda acc: acc.Reset())
      self.assertEqual(0.0, layer1.accumulators.acc1.GetValue().eval())
      self.assertEqual(0.0, layer1.layer1a.accumulators.acc2.GetValue().eval())
      self.assertEqual(
          0.0,
          layer1.layer1b.layer1b1.accumulators.acc3.GetValue().eval())

      # Set and check.
      layer1.SetAccumulatorValues(saved_pack)
      self.assertEqual(1.0, layer1.accumulators.acc1.GetValue().eval())
      self.assertEqual(2.0, layer1.layer1a.accumulators.acc2.GetValue().eval())
      self.assertEqual(
          3.0,
          layer1.layer1b.layer1b1.accumulators.acc3.GetValue().eval())

  def testAddFunction(self):
    layer_p = base_layer.BaseLayer.Params()
    layer_p.name = 'test'
    layer = layer_p.Instantiate()

    layer.AddFunction('test1', lambda: 1)
    with self.assertRaises(AttributeError):
      layer.AddFunction('test1', lambda: 2)
    self.assertEquals(1, layer.fns.test1())

  def testAttributeErrorInPropertyGetter(self):

    class BadLayer(base_layer.BaseLayer):

      @classmethod
      def Params(cls):
        return super(BadLayer, cls).Params()

      @base_layer.initializer
      def __init__(self, params):
        super(BadLayer, self).__init__(params)

      @property
      def bad_property(self):
        raise AttributeError('INTERNAL')

    layer_p = BadLayer.Params()
    layer_p.name = 'test'
    layer = layer_p.Instantiate()

    with self.assertRaisesRegex(AttributeError, 'bad_sub_layer'):
      _ = layer.bad_sub_layer

    with self.assertRaisesRegex(AttributeError, 'INTERNAL'):
      _ = layer.bad_property

  def testIsLayerParams(self):
    self.assertTrue(base_layer.IsLayerParams(base_layer.BaseLayer.Params()))
    self.assertTrue(base_layer.IsLayerParams(TestLayer.Params()))
    self.assertFalse(base_layer.IsLayerParams(None))
    self.assertFalse(base_layer.IsLayerParams(hyperparams.Params()))
    self.assertFalse(
        base_layer.IsLayerParams(
            hyperparams.InstantiableParams(base_layer.Accumulator)))

if __name__ == '__main__':
  tf.test.main()
<commit_msg>add test for DefaultVN and DisableVN<commit_after># Lint as: python2, python3
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for base_layer."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import lingvo.compat as tf
from lingvo.core import base_layer
from lingvo.core import hyperparams
from lingvo.core import py_utils
from lingvo.core import test_utils

class AddingAccumulator(base_layer.Accumulator):

  def DefaultValue(self):
    return tf.convert_to_tensor(0.0)

  def Update(self, new_value):
    self.SetValue(self.GetValue() + new_value)

def EvalAndFlatten(nmap):
  return nmap.Transform(lambda x: x.eval()).FlattenItems()

class TestLayer(base_layer.BaseLayer):

  @base_layer.initializer
  def __init__(self, params):
    super(TestLayer, self).__init__(params)
    p = self.params
    with tf.variable_scope(p.name):
      self.CreateVariable(
          'w',
          py_utils.WeightParams(
              shape=[4, 4],
              dtype=p.dtype,
              init=p.params_init,
              collections=[self.__class__.__name__ + '_vars']))
      self.CreateVariable(
          'b',
          py_utils.WeightParams(
              shape=[4],
              dtype=p.dtype,
              init=py_utils.WeightInit.Constant(),
              collections=[
                  self.__class__.__name__ + '_vars',
                  py_utils.SKIP_LP_REGULARIZATION
              ]))

class BaseLayerTest(test_utils.TestCase):

  def testCopyBaseParams(self):
    # CopyBaseParams should only overwrite vn setting when target use
    # default vn config.
    layer_base_p = base_layer.BaseLayer.Params()
    from_param = layer_base_p.Copy()
    to_param = layer_base_p.Copy()
    from_param.vn.global_vn = True
    from_param.random_seed = 1234
    from_param.skip_lp_regularization = True
    # Target use default, overwrite.
    base_layer.BaseLayer.CopyBaseParams(from_param, to_param)
    self.assertTrue(to_param.vn.global_vn)
    self.assertEqual(1234, to_param.random_seed)
    self.assertTrue(to_param.skip_lp_regularization)
    to_param = layer_base_p.Copy()
    to_param.vn.per_step_vn = True
    to_param.random_seed = 4321
    to_param.skip_lp_regularization = False
    # Target does not use default, should not overwrite.
    base_layer.BaseLayer.CopyBaseParams(from_param, to_param)
    self.assertTrue(to_param.vn.per_step_vn)
    self.assertFalse(to_param.vn.global_vn)
    self.assertEqual(4321, to_param.random_seed)
    self.assertFalse(to_param.skip_lp_regularization)

  def testCreateChildren(self):
    layer_p = base_layer.BaseLayer.Params()
    layer_p.name = 'test'
    layer = layer_p.Instantiate()
    layer.CreateChildren('a', [layer_p, [layer_p, layer_p], layer_p])
    self.assertEqual(len(layer.a), 3)
    self.assertEqual(len(layer.a[1]), 2)
    self.assertEqual(len(layer.vars.a), 3)
    self.assertEqual(len(layer.vars.a[1]), 2)
    self.assertEqual(len(layer.theta.a), 3)
    self.assertEqual(len(layer.theta.a[1]), 2)

  def testCreateVariable(self):
    layer_p = TestLayer.Params().Set(name='test')
    layer = layer_p.Instantiate()
    self.assertEqual('test/w/var:0', layer.vars.w.name)
    self.assertEqual('test/b/var:0', layer.vars.b.name)
    self.assertNotIn(layer.vars.w,
                     tf.get_collection(py_utils.SKIP_LP_REGULARIZATION))
    # 'b' always skips Lp regularization.
    self.assertIn(layer.vars.b,
                  tf.get_collection(py_utils.SKIP_LP_REGULARIZATION))

  def testCreateVariableSkipLpRegularization(self):
    layer_p = TestLayer.Params().Set(name='test', skip_lp_regularization=True)
    layer = layer_p.Instantiate()
    self.assertIn(layer.vars.w,
                  tf.get_collection(py_utils.SKIP_LP_REGULARIZATION))
    self.assertIn(layer.vars.b,
                  tf.get_collection(py_utils.SKIP_LP_REGULARIZATION))

  def testGetDescendant(self):
    q = base_layer.BaseLayer.Params()
    q.name = 'test'
    l = q.Instantiate()
    p = base_layer.BaseLayer.Params()
    l.CreateChild('a', p)
    l.CreateChild('b', p)
    l.a.CreateChild('c', p)
    l.a.c.CreateChild('d', p)
    l.b.CreateChild('e', p)
    self.assertEqual(l, l.GetDescendant(''))
    self.assertEqual(l.a, l.GetDescendant('a'))
    self.assertEqual(l.b, l.GetDescendant('b'))
    self.assertEqual(l.a.c, l.GetDescendant('a.c'))
    self.assertEqual(l.a.c.d, l.GetDescendant('a.c.d'))
    self.assertEqual(l.b.e, l.GetDescendant('b.e'))

  def testCreateAccumulator(self):
    layer_p = base_layer.BaseLayer.Params()
    layer_p.name = 'test'
    layer = layer_p.Instantiate()
    layer.CreateChild('child', layer_p)

    # First accumulator should succeed.
    acc1 = AddingAccumulator()
    layer.RegisterAccumulator('acc1', acc1)

    # Name of existing child should fail.
    with self.assertRaises(AssertionError):
      layer.RegisterAccumulator('child', AddingAccumulator())

    # Duplicate should fail.
    with self.assertRaises(AssertionError):
      layer.RegisterAccumulator('acc1', AddingAccumulator())

    # Child with the same name should fail.
    with self.assertRaises(AssertionError):
      layer.CreateChild('acc1', layer_p)

    self.assertEqual(acc1, layer.accumulators.acc1)

    # Get of not created accumulator should fail.
    with self.assertRaises(AttributeError):
      layer.accumulators.notexist

  def testGetUpdateAccumulator(self):
    with self.session():
      layer_p = base_layer.BaseLayer.Params()
      layer_p.name = 'test'
      layer = layer_p.Instantiate()

      layer.RegisterAccumulator('acc1', AddingAccumulator())

      # Initial value.
      self.assertEqual(0.0, layer.accumulators.acc1.GetValue().eval())

      # Update/merge.
      layer.accumulators.acc1.Update(1.0)
      layer.accumulators.acc1.Update(1.0)
      self.assertEqual(2.0, layer.accumulators.acc1.GetValue().eval())

      # Reset.
      layer.accumulators.Transform(lambda acc: acc.Reset())
      self.assertEqual(0.0, layer.accumulators.acc1.GetValue().eval())

  def testAccumulatorDisableEnable(self):
    with self.session():
      layer_p = base_layer.BaseLayer.Params()
      layer_p.name = 'test'
      layer = layer_p.Instantiate()

      layer.RegisterAccumulator('acc1', AddingAccumulator())
      layer.accumulators.acc1.Update(1.0)

      # Disable should force value to 0 and reject updates.
      layer.accumulators.Transform(lambda acc: acc.Disable())
      self.assertEqual(0.0, layer.accumulators.acc1.GetValue().eval())
      layer.accumulators.acc1.Update(3.0)
      self.assertEqual(0.0, layer.accumulators.acc1.GetValue().eval())
      layer.accumulators.Transform(lambda acc: acc.Enable())

      # Should restore.
      self.assertEqual(1.0, layer.accumulators.acc1.GetValue().eval())

  def testGetSetAccumulatorValues(self):
    with self.session():
      layer_p = base_layer.BaseLayer.Params()
      layer_p.name = 'test'
      layer1 = layer_p.Instantiate()
      layer1.CreateChild('layer1a', layer_p)
      layer1.CreateChild('layer1b', layer_p)
      layer1.layer1b.CreateChild('layer1b1', layer_p)

      # Create nested accumulators:
      #   acc1: layer1
      #   acc2: layer1.layer1a
      #   acc3: layer1.layer1b.layer1b1
      layer1.RegisterAccumulator('acc1', AddingAccumulator())
      layer1.layer1a.RegisterAccumulator('acc2', AddingAccumulator())
      layer1.layer1b.layer1b1.RegisterAccumulator('acc3', AddingAccumulator())

      # Pack with initial values.
      initial_pack = layer1.GetAccumulatorValues()
      initial_pack_eval = EvalAndFlatten(initial_pack)
      print('Initial values pack =', initial_pack_eval)
      self.assertEqual(initial_pack_eval, [('acc1', 0.0), ('layer1a.acc2', 0.0),
                                           ('layer1b.layer1b1.acc3', 0.0)])

      # Update to a new known state.
      layer1.accumulators.acc1.Update(1.0)
      layer1.layer1a.accumulators.acc2.Update(2.0)
      layer1.layer1b.layer1b1.accumulators.acc3.Update(3.0)
      updated_pack = layer1.GetAccumulatorValues()
      updated_pack_eval = EvalAndFlatten(updated_pack)
      print('Updated values pack =', updated_pack_eval)
      self.assertEqual(updated_pack_eval, [('acc1', 1.0), ('layer1a.acc2', 2.0),
                                           ('layer1b.layer1b1.acc3', 3.0)])

      # Save and reset.
      saved_pack = layer1.GetAccumulatorValues()
      layer1.accumulators.Transform(lambda acc: acc.Reset())
      self.assertEqual(0.0, layer1.accumulators.acc1.GetValue().eval())
      self.assertEqual(0.0, layer1.layer1a.accumulators.acc2.GetValue().eval())
      self.assertEqual(
          0.0,
          layer1.layer1b.layer1b1.accumulators.acc3.GetValue().eval())

      # Set and check.
      layer1.SetAccumulatorValues(saved_pack)
      self.assertEqual(1.0, layer1.accumulators.acc1.GetValue().eval())
      self.assertEqual(2.0, layer1.layer1a.accumulators.acc2.GetValue().eval())
      self.assertEqual(
          3.0,
          layer1.layer1b.layer1b1.accumulators.acc3.GetValue().eval())

  def testAddFunction(self):
    layer_p = base_layer.BaseLayer.Params()
    layer_p.name = 'test'
    layer = layer_p.Instantiate()

    layer.AddFunction('test1', lambda: 1)
    with self.assertRaises(AttributeError):
      layer.AddFunction('test1', lambda: 2)
    self.assertEquals(1, layer.fns.test1())

  def testAttributeErrorInPropertyGetter(self):

    class BadLayer(base_layer.BaseLayer):

      @classmethod
      def Params(cls):
        return super(BadLayer, cls).Params()

      @base_layer.initializer
      def __init__(self, params):
        super(BadLayer, self).__init__(params)

      @property
      def bad_property(self):
        raise AttributeError('INTERNAL')

    layer_p = BadLayer.Params()
    layer_p.name = 'test'
    layer = layer_p.Instantiate()

    with self.assertRaisesRegex(AttributeError, 'bad_sub_layer'):
      _ = layer.bad_sub_layer

    with self.assertRaisesRegex(AttributeError, 'INTERNAL'):
      _ = layer.bad_property

  def testIsLayerParams(self):
    self.assertTrue(base_layer.IsLayerParams(base_layer.BaseLayer.Params()))
    self.assertTrue(base_layer.IsLayerParams(TestLayer.Params()))
    self.assertFalse(base_layer.IsLayerParams(None))
    self.assertFalse(base_layer.IsLayerParams(hyperparams.Params()))
    self.assertFalse(
        base_layer.IsLayerParams(
            hyperparams.InstantiableParams(base_layer.Accumulator)))

  def testDefaultVnParams(self):
    default_vn = base_layer.DefaultVN()
    disable_vn = py_utils.DisableVN()
    self.assertNotEqual(default_vn, disable_vn)

if __name__ == '__main__':
  tf.test.main()
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
# Copyright 2014-2017 Luc Saffre
# License: BSD (see file COPYING for details)
"""
Base Django settings for Lino Tera applications.

"""

from __future__ import print_function
from __future__ import unicode_literals

from lino.projects.std.settings import *
from lino.api.ad import _
from lino_tera import SETUP_INFO

class Site(Site):

    verbose_name = "Lino Tera"
    version = SETUP_INFO['version']
    url = "http://tera.lino-framework.org/"

    demo_fixtures = 'std demo minimal_ledger euvatrates demo2'.split()

    demo_fixtures = ['std', 'demo', 'demo2']
                     # 'linotickets',
                     # 'tractickets', 'luc']

    project_model = 'tera.Client'
    # project_model = 'contacts.Partner'
    textfield_format = 'html'
    user_types_module = 'lino_tera.lib.tera.user_types'
    workflows_module = 'lino_tera.lib.tera.workflows'
    custom_layouts_module = 'lino_tera.lib.tera.layouts'
    obj2text_template = "**{0}**"

    default_build_method = 'appyodt'

    # experimental use of rest_framework:
    # root_urlconf = 'lino_book.projects.team.urls'

    # migration_class = 'lino_tera.lib.tera.migrate.Migrator'

    auto_configure_logger_names = "atelier django lino lino_xl lino_noi"

    def get_installed_apps(self):
        yield super(Site, self).get_installed_apps()
        yield 'lino.modlib.gfks'
        # yield 'lino.modlib.users'
        yield 'lino_tera.lib.users'
        yield 'lino_xl.lib.countries'
        yield 'lino_xl.lib.properties'
        yield 'lino_tera.lib.contacts'
        yield 'lino_tera.lib.households'
        yield 'lino_xl.lib.coachings'
        # yield 'lino_tera.lib.lists'
        yield 'lino_xl.lib.beid'
        # yield 'lino_xl.lib.addresses'
        yield 'lino_xl.lib.humanlinks',
        yield 'lino_voga.lib.products'
        yield 'lino_tera.lib.courses'
        # yield 'lino_xl.lib.accounts'
        yield 'lino_xl.lib.sales'
        # yield 'lino_xl.lib.vat'
        yield 'lino_xl.lib.sepa'
        yield 'lino_xl.lib.finan'
        yield 'lino_xl.lib.invoicing'
        yield 'lino_xl.lib.bevats'
        yield 'lino_xl.lib.ana'
        # 'lino_xl.lib.projects',
        yield 'lino_xl.lib.blogs'
        yield 'lino_xl.lib.notes'
        # yield 'lino_tera.lib.tickets'
        yield 'lino_xl.lib.faculties'
        # yield 'lino_xl.lib.votes'
        # yield 'lino_tera.lib.clocking'
        # yield 'lino_xl.lib.deploy'
        # yield 'lino_presto.lib.clocking'
        # yield 'lino.modlib.uploads'
        yield 'lino_xl.lib.extensible'
        yield 'lino_xl.lib.cal'
        # yield 'lino_xl.lib.outbox'
        yield 'lino_xl.lib.excerpts'
        yield 'lino_xl.lib.appypod'
        # yield 'lino_xl.lib.postings'
        # yield 'lino_xl.lib.pages'

        yield 'lino.modlib.export_excel'
        yield 'lino.modlib.plausibility'
        yield 'lino.modlib.tinymce'
        # yield 'lino.modlib.wkhtmltopdf'
        yield 'lino.modlib.weasyprint'

        yield 'lino_tera.lib.tera'
        yield 'lino_tera.lib.teams'

    def setup_plugins(self):
        super(Site, self).setup_plugins()
        self.plugins.countries.configure(country_code='BE')
        # self.plugins.clocking.configure(ticket_model='contacts.Person')
        self.plugins.faculties.configure(demander_model='contacts.Person')
        # self.plugins.coachings.configure(client_model='tera.Client')
        self.plugins.coachings.configure(client_model='contacts.Partner')
        self.plugins.countries.configure(hide_region=True)
        self.plugins.countries.configure(country_code='BE')
        self.plugins.ledger.configure(start_year=2015)
        self.plugins.ledger.configure(use_pcmn=True)
        self.plugins.courses.configure(teacher_model='users.User')
        self.plugins.courses.configure(pupil_model='contacts.Person')

# the following line should not be active in a checked-in version
#~ DATABASES['default']['NAME'] = ':memory:'

USE_TZ = True
# TIME_ZONE = 'Europe/Brussels'
# TIME_ZONE = 'Europe/Tallinn'
TIME_ZONE = 'UTC'

<commit_msg>http://luc.lino-framework.org/blog/2017/0810.html<commit_after># -*- coding: UTF-8 -*-
# Copyright 2014-2017 Luc Saffre
# License: BSD (see file COPYING for details)
"""
Base Django settings for Lino Tera applications.

"""

from __future__ import print_function
from __future__ import unicode_literals

from lino.projects.std.settings import *
from lino.api.ad import _
from lino_tera import SETUP_INFO

class Site(Site):

    verbose_name = "Lino Tera"
    version = SETUP_INFO['version']
    url = "http://tera.lino-framework.org/"

    demo_fixtures = 'std demo minimal_ledger euvatrates demo2'.split()

    demo_fixtures = ['std', 'demo', 'demo2']
                     # 'linotickets',
                     # 'tractickets', 'luc']

    # project_model = 'tera.Client'
    project_model = 'courses.Course'
    # project_model = 'contacts.Partner'
    textfield_format = 'html'
    user_types_module = 'lino_tera.lib.tera.user_types'
    workflows_module = 'lino_tera.lib.tera.workflows'
    custom_layouts_module = 'lino_tera.lib.tera.layouts'
    obj2text_template = "**{0}**"

    default_build_method = 'appyodt'

    # experimental use of rest_framework:
    # root_urlconf = 'lino_book.projects.team.urls'

    # migration_class = 'lino_tera.lib.tera.migrate.Migrator'

    auto_configure_logger_names = "atelier django lino lino_xl lino_noi"

    def get_installed_apps(self):
        yield super(Site, self).get_installed_apps()
        yield 'lino.modlib.gfks'
        # yield 'lino.modlib.users'
        yield 'lino_tera.lib.users'
        yield 'lino_xl.lib.countries'
        yield 'lino_xl.lib.properties'
        yield 'lino_tera.lib.contacts'
        yield 'lino_tera.lib.households'
        yield 'lino_xl.lib.coachings'
        # yield 'lino_tera.lib.lists'
        yield 'lino_xl.lib.beid'
        # yield 'lino_xl.lib.addresses'
        yield 'lino_xl.lib.humanlinks',
        yield 'lino_voga.lib.products'
        yield 'lino_tera.lib.courses'
        # yield 'lino_xl.lib.accounts'
        yield 'lino_xl.lib.sales'
        # yield 'lino_xl.lib.vat'
        yield 'lino_xl.lib.sepa'
        yield 'lino_xl.lib.finan'
        yield 'lino_xl.lib.invoicing'
        yield 'lino_xl.lib.bevats'
        yield 'lino_xl.lib.ana'
        # 'lino_xl.lib.projects',
        yield 'lino_xl.lib.blogs'
        yield 'lino_xl.lib.notes'
        # yield 'lino_tera.lib.tickets'
        yield 'lino_xl.lib.faculties'
        # yield 'lino_xl.lib.votes'
        # yield 'lino_tera.lib.clocking'
        # yield 'lino_xl.lib.deploy'
        # yield 'lino_presto.lib.clocking'
        # yield 'lino.modlib.uploads'
        yield 'lino_xl.lib.extensible'
        yield 'lino_xl.lib.cal'
        # yield 'lino_xl.lib.outbox'
        yield 'lino_xl.lib.excerpts'
        yield 'lino_xl.lib.appypod'
        # yield 'lino_xl.lib.postings'
        # yield 'lino_xl.lib.pages'

        yield 'lino.modlib.export_excel'
        yield 'lino.modlib.plausibility'
        yield 'lino.modlib.tinymce'
        # yield 'lino.modlib.wkhtmltopdf'
        yield 'lino.modlib.weasyprint'

        yield 'lino_tera.lib.tera'
        yield 'lino_tera.lib.teams'

    def setup_plugins(self):
        super(Site, self).setup_plugins()
        self.plugins.countries.configure(country_code='BE')
        # self.plugins.clocking.configure(ticket_model='contacts.Person')
        self.plugins.faculties.configure(demander_model='contacts.Person')
        # self.plugins.coachings.configure(client_model='tera.Client')
        self.plugins.coachings.configure(client_model='contacts.Partner')
        self.plugins.countries.configure(hide_region=True)
        self.plugins.countries.configure(country_code='BE')
        self.plugins.ledger.configure(start_year=2015)
        self.plugins.ledger.configure(use_pcmn=True)
        self.plugins.courses.configure(teacher_model='users.User')
        self.plugins.courses.configure(pupil_model='contacts.Person')

# the following line should not be active in a checked-in version
#~ DATABASES['default']['NAME'] = ':memory:'

USE_TZ = True
# TIME_ZONE = 'Europe/Brussels'
# TIME_ZONE = 'Europe/Tallinn'
TIME_ZONE = 'UTC'

<|endoftext|>


<commit_before>'''
Venn diagram plotting routines.
Rounded square, nested venn plotter.

Copyright 2013, Johannes Buchner.

Licensed under MIT license.
'''
import numpy
import warnings
import string

from matplotlib_venn import venn2, venn3
import matplotlib.pyplot as plt
import matplotlib.transforms as mtransforms
from matplotlib.patches import FancyBboxPatch
import itertools

default_attrs = [dict(ec=c, fc='None', alpha=0.7) for c in ['black', 'blue', 'red', 'green', 'magenta']]
default_attrs = itertools.cycle(default_attrs)

def plot_box(bbox, label='', attrs={}):
	((xmin, ymin), (xmax, ymax)) = bbox.get_points()
	if attrs is None:
		attrs = default_attrs.next()
		print 'picked attrs:', attrs

	ax = plt.gca()
	print 'plotting %s at' % label, ((xmin, ymin), (xmax, ymax))
	p_fancy = FancyBboxPatch((xmin, ymin),
		xmax - xmin, ymax - ymin,
		boxstyle="round,pad=0.0, rounding_size=0.02",
		**attrs)
	ax.add_patch(p_fancy)
	ax.text(xmin + 0.01, ymax - 0.01, label,
		horizontalalignment='left',
		verticalalignment='top',
		)

def nodesets_rectangles((node, children), bbox):
	# plot what the children make out in the bbox
	nodesize, nodelabel, nodeattrs = node
	nchildren = len(children)

	print 'node', node
	print '   has %d children:' % nchildren
	for i, c in enumerate(children):
		print '     %i: %s' % (i, c)
	((xmin0, ymin0), (xmax0, ymax0)) = bbox.get_points()
	if nchildren > 0:
		deltay = ymax0 - ymin0
		deltax = xmax0 - xmin0
		nsizechildren = sum([size for (size, label, attrs), grandchildren in children])
		empty = 1 - nsizechildren * 1. / nodesize
		fbuffer = (empty)**0.5
		yratio = deltay * (nsizechildren * 1. / nodesize) / deltax
		#yratio = deltax / deltax * (nsizechildren * 1. / nodesize)

		#padding = arearatio**0.5 / 10
		#shorteningratio =
		#fbuffer = 0
		print 'yratio %s:' % str(node), yratio
		print 'deltax, deltay:', deltax, deltay
	for child, grandchildren in children:
		((xmin, ymin), (xmax, ymax)) = bbox.get_points()

		size, label, attrs = child
		arearatio = size * 1. / nodesize
		print 'arearatio of child:', arearatio

		if nchildren == 1:
			# update borders, proportional to area
			f = (arearatio * 8/3.)**0.5
			fx = 1/2. * f
			fy = 3/4. * f
			f = arearatio**0.5
			ymax = ymax - deltay * (1 - fx*0.95)
			xmin = xmin + deltax * (1 - fy*0.95)
			ymin = ymin + deltay * (fx*0.05)
			xmax = xmax - deltax * (fy*0.05)
		else:

			# update borders, split dependent on xratio
			# we prefer to split in y (because label text flows in x)
			# but split if ratio is too extreme
			if yratio > 0.4: # split in y
				print 'splitting in y: starting box', ((xmin0, ymin0), (xmax0, ymax0))
				#ymin,  ymax  = ymax0 - deltay * arearatio + 0.*deltay * fbuffer / 2, ymax0
				#ymax0, ymin0 = ymax0 - deltay * arearatio - 0.*deltay * fbuffer / 2, ymin0
				ymin,  ymax  = ymin0, ymin0 + deltay * arearatio - 1.*deltay * fbuffer / 40
				ymax0, ymin0 = ymax0, ymin0 + deltay * arearatio + 1.*deltay * fbuffer / 40
				#ymax0 = ymax0 - deltay * arearatio - deltay * fbuffer / 2
				#ymin = ymax0 + deltay * fbuffer / 2
				#xmax, xmin = xmax0, xmin0
				print 'splitting in y: child box', ((xmin, ymin), (xmax, ymax))
				print 'splitting in y: remaining box', ((xmin0, ymin0), (xmax0, ymax0))
			else:
				print 'splitting in x: starting box', ((xmin0, ymin0), (xmax0, ymax0))
				#xmin,  xmax  = xmin0, xmin0 + deltay * arearatio - 1.*deltax * fbuffer / 80
				xmin,  xmax  = xmax0 - deltax * arearatio + deltax * fbuffer / 40, xmax0
				xmax0, xmin0 = xmax0 - deltax * arearatio - deltax * fbuffer / 40, xmin0

				print 'splitting in x: child box', ((xmin, ymin), (xmax, ymax))
				print 'splitting in x: remaining box', ((xmin0, ymin0), (xmax0, ymax0))
		# recurse
		childbox = mtransforms.Bbox(((xmin, ymin), (xmax, ymax)))
		nodesets_rectangles((child, grandchildren), childbox)
		plot_box(bbox=childbox, attrs=attrs, label=label)

def treesets_rectangles(tree):
	((xmin, xmax), (ymin, ymax)) = [(0, 1), (0, 1)]
	superset = None
	ax = plt.gca()
	ax.set_aspect(1.)
	ax.set_xlim(-0.1, 1.1)
	ax.set_ylim(-0.1, 1.1)
	ax.set_xticks([])
	ax.set_yticks([])
	plt.axis('off')
	# start with plotting root node
	root, children = tree
	size, label, attrs = root
	rootbox = mtransforms.Bbox([[xmin, ymin], [xmax, ymax]])
	nodesets_rectangles((root, children), rootbox)
	plot_box(bbox=rootbox, attrs=attrs, label=label)

def nestedsets_rectangles(setsizes, labels = None, attrs = None):
	nsets = len(setsizes)
	if labels is None:
		labels = list(string.ascii_uppercase)[:nsets]
	if attrs is None:
		attrs = [default_attrs.next() for i in range(nsets)]
		#itertools.cycle([dict(ec=c, fc='None', alpha=0.7)
		#	for c in ['black', 'blue', 'red', 'green', 'magenta']])

	tree = []
	for node in list(zip(setsizes, labels, attrs))[::-1]:
		tree = [[node, tree]]
	treesets_rectangles(tree[0])

<commit_msg>resolved a issue with empty children, and where the area was not preserved<commit_after>'''
Venn diagram plotting routines.
Rounded square, nested venn plotter.

Copyright 2013, Johannes Buchner.

Licensed under MIT license.
'''
import numpy
import warnings
import string

from matplotlib_venn import venn2, venn3
import matplotlib.pyplot as plt
import matplotlib.transforms as mtransforms
from matplotlib.patches import FancyBboxPatch
import itertools
import scipy.stats

default_attrs = [dict(ec=c, fc='None', alpha=0.7) for c in ['black', 'blue', 'red', 'green', 'magenta']]
default_attrs = itertools.cycle(default_attrs)

def plot_box(bbox, label='', attrs={}):
	((xmin, ymin), (xmax, ymax)) = bbox.get_points()
	if attrs is None:
		attrs = default_attrs.next()
		print 'picked attrs:', attrs

	ax = plt.gca()
	print 'plotting %s at' % label, ((xmin, ymin), (xmax, ymax))
	p_fancy = FancyBboxPatch((xmin, ymin),
		xmax - xmin, ymax - ymin,
		boxstyle="round,pad=0.0, rounding_size=0.02",
		**attrs)
	ax.add_patch(p_fancy)
	ax.text(xmin + 0.01, ymax - 0.01, label,
		horizontalalignment='left',
		verticalalignment='top',
		)

def nodesets_rectangles((node, children), bbox):
	# plot what the children make out in the bbox
	nodesize, nodelabel, nodeattrs = node

	nchildren = len(children)

	print 'node', node
	print '   has %d children:' % nchildren
	for i, c in enumerate(children):
		print '     %i: %s' % (i, c)
	((xmin0, ymin0), (xmax0, ymax0)) = bbox.get_points()
	deltay = ymax0 - ymin0
	deltax = xmax0 - xmin0
	if nchildren > 0:
		nsizechildren = sum([size for (size, label, attrs), grandchildren in children])
		empty = 1 - nsizechildren * 1. / nodesize
		fbuffer = (empty)**0.5
		yratio = deltay * (nsizechildren * 1. / nodesize) / deltax
		#yratio = deltax / deltax * (nsizechildren * 1. / nodesize)

		#padding = arearatio**0.5 / 10
		#shorteningratio =
		#fbuffer = 0
		print 'yratio %s:' % str(node), yratio
		print 'deltax, deltay:', deltax, deltay
	for child, grandchildren in children:
		((xmin, ymin), (xmax, ymax)) = bbox.get_points()

		size, label, attrs = child
		arearatio = size * 1. / nodesize
		print 'arearatio of child:', arearatio
		if arearatio == 0:
			# skip empty children
			continue

		if nchildren == 1:
			# update borders, proportional to area
			print 'single subset: arearatio', arearatio
			# along a beta distribution; cdf is area
			a, b = 10, 1
			rv = scipy.stats.beta(a, b)
			fx = rv.ppf(arearatio)
			fy = arearatio / fx
			print 'fx, fy:', fx, fy
			# add padding if possible
			ypad = min(fy*0.02, 1 - fy)
			xpad = min(fx*0.02, 1 - fx)

			ymax = ymax - deltay * (1 - (fy + ypad))
			xmin = xmin + deltax * (1 - (fx + xpad))
			ymin = ymin + deltay * ypad
			xmax = xmax - deltax * xpad
		else:
			# update borders, split dependent on xratio
			# we prefer to split in y (because label text flows in x)
			# but split if ratio is too extreme
			if yratio > 0.4: # split in y
				print 'splitting in y: starting box', ((xmin0, ymin0), (xmax0, ymax0))
				#ymin,  ymax  = ymax0 - deltay * arearatio + 0.*deltay * fbuffer / 2, ymax0
				#ymax0, ymin0 = ymax0 - deltay * arearatio - 0.*deltay * fbuffer / 2, ymin0
				ymin,  ymax  = ymin0, ymin0 + deltay * arearatio - 1.*deltay * fbuffer / 40
				ymax0, ymin0 = ymax0, ymin0 + deltay * arearatio + 1.*deltay * fbuffer / 40
				#ymax0 = ymax0 - deltay * arearatio - deltay * fbuffer / 2
				#ymin = ymax0 + deltay * fbuffer / 2
				#xmax, xmin = xmax0, xmin0
				print 'splitting in y: child box', ((xmin, ymin), (xmax, ymax))
				print 'splitting in y: remaining box', ((xmin0, ymin0), (xmax0, ymax0))
			else:
				print 'splitting in x: starting box', ((xmin0, ymin0), (xmax0, ymax0))
				#xmin,  xmax  = xmin0, xmin0 + deltay * arearatio - 1.*deltax * fbuffer / 80
				xmin,  xmax  = xmax0 - deltax * arearatio + deltax * fbuffer / 40, xmax0
				xmax0, xmin0 = xmax0 - deltax * arearatio - deltax * fbuffer / 40, xmin0

				print 'splitting in x: child box', ((xmin, ymin), (xmax, ymax))
				print 'splitting in x: remaining box', ((xmin0, ymin0), (xmax0, ymax0))
		# recurse
		childbox = mtransforms.Bbox(((xmin, ymin), (xmax, ymax)))
		nodesets_rectangles((child, grandchildren), childbox)
		plot_box(bbox=childbox, attrs=attrs, label=label)

def treesets_rectangles(tree):
	((xmin, xmax), (ymin, ymax)) = [(0, 1), (0, 1)]
	superset = None
	ax = plt.gca()
	ax.set_aspect(1.)
	ax.set_xlim(-0.1, 1.1)
	ax.set_ylim(-0.1, 1.1)
	ax.set_xticks([])
	ax.set_yticks([])
	plt.axis('off')
	# start with plotting root node
	root, children = tree
	size, label, attrs = root
	assert size > 0
	rootbox = mtransforms.Bbox([[xmin, ymin], [xmax, ymax]])
	nodesets_rectangles((root, children), rootbox)
	plot_box(bbox=rootbox, attrs=attrs, label=label)

def nestedsets_rectangles(setsizes, labels = None, attrs = None):
	nsets = len(setsizes)
	if labels is None:
		labels = list(string.ascii_uppercase)[:nsets]
	if attrs is None:
		attrs = [default_attrs.next() for i in range(nsets)]
		#itertools.cycle([dict(ec=c, fc='None', alpha=0.7)
		#	for c in ['black', 'blue', 'red', 'green', 'magenta']])

	tree = []
	for node in list(zip(setsizes, labels, attrs))[::-1]:
		tree = [[node, tree]]
	treesets_rectangles(tree[0])

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from ast import literal_eval
from lxml import etree

from odoo.osv import orm
from odoo import models, fields, api, _
from odoo.exceptions import ValidationError

class ProductConfigurator(models.TransientModel):
    _inherit = 'product.configurator'

    product_id = fields.Many2one(readonly=False)
    search_filter = fields.Char('Search Code')
    manufacturer_id = fields.Many2one(
        'product.attribute.value', string='Manufacturer', domain=lambda self: [
            ('attribute_id', '=', self.env.ref('product_configurator_search.attribute_manufacturer').id)])
    allowed_product_ids = fields.Many2many(
        'product.product', string='Allowed Products', compute='_compute_allowed_product_ids')

    @api.multi
    @api.onchange('product_id')
    def _onchange_product_id(self):
        attribute_manufacturer = self.env.ref('product_configurator_search.attribute_manufacturer')
        for record in self:
            if record.product_id:
                record.product_tmpl_id = record.product_id.product_tmpl_id
                manufacturer_id = record.product_id.attribute_value_ids.filtered(
                    lambda x: x.attribute_id == attribute_manufacturer)
                record.manufacturer_id = manufacturer_id

    @api.multi
    @api.onchange('search_filter')
    def _onchange_search_filter(self):
        self.ensure_one()
        # allowed_product_ids updated by search_filter
        if self.search_filter:
            self.product_id = self.allowed_product_ids and self.allowed_product_ids[0] or False
            return {'domain': {'product_id': [('id', 'in', self.allowed_product_ids.ids)]}}
        else:
            return {'domain': {'product_id': [('config_ok', '=', True)]}}

    @api.multi
    @api.depends('search_filter')
    def _compute_allowed_product_ids(self):
        self.ensure_one()
        attribute_mpn = self.env.ref('product_configurator_search.attribute_mpn')
        matching_values = self.env['product.attribute.value'].search([
            ('attribute_id', '=', attribute_mpn.id),
            ('name', 'like', self.search_filter),
        ])
        self.allowed_product_ids = matching_values.mapped('product_ids').filtered('config_ok')
<commit_msg>product_configurator_search - rename MPN search<commit_after># -*- coding: utf-8 -*-

from ast import literal_eval
from lxml import etree

from odoo.osv import orm
from odoo import models, fields, api, _
from odoo.exceptions import ValidationError

class ProductConfigurator(models.TransientModel):
    _inherit = 'product.configurator'

    product_id = fields.Many2one(readonly=False)
    search_filter = fields.Char('MPN')
    manufacturer_id = fields.Many2one(
        'product.attribute.value', string='Manufacturer', domain=lambda self: [
            ('attribute_id', '=', self.env.ref('product_configurator_search.attribute_manufacturer').id)])
    allowed_product_ids = fields.Many2many(
        'product.product', string='Allowed Products', compute='_compute_allowed_product_ids')

    @api.multi
    @api.onchange('product_id')
    def _onchange_product_id(self):
        attribute_manufacturer = self.env.ref('product_configurator_search.attribute_manufacturer')
        for record in self:
            if record.product_id:
                record.product_tmpl_id = record.product_id.product_tmpl_id
                manufacturer_id = record.product_id.attribute_value_ids.filtered(
                    lambda x: x.attribute_id == attribute_manufacturer)
                record.manufacturer_id = manufacturer_id

    @api.multi
    @api.onchange('search_filter')
    def _onchange_search_filter(self):
        self.ensure_one()
        # allowed_product_ids updated by search_filter
        if self.search_filter:
            self.product_id = self.allowed_product_ids and self.allowed_product_ids[0] or False
            return {'domain': {'product_id': [('id', 'in', self.allowed_product_ids.ids)]}}
        else:
            return {'domain': {'product_id': [('config_ok', '=', True)]}}

    @api.multi
    @api.depends('search_filter')
    def _compute_allowed_product_ids(self):
        self.ensure_one()
        attribute_mpn = self.env.ref('product_configurator_search.attribute_mpn')
        matching_values = self.env['product.attribute.value'].search([
            ('attribute_id', '=', attribute_mpn.id),
            ('name', 'like', self.search_filter),
        ])
        self.allowed_product_ids = matching_values.mapped('product_ids').filtered('config_ok')
<|endoftext|>


<commit_before>"""
Tests for the CSV file sniffer
"""
# Author: Ilan Schnell <ischnell@enthought.com>
# Copyright (c) 2008, Enthought, Inc.
# License: BSD Style.
import glob
import os
import os.path
import sys
import unittest
from test.test_support import TESTFN, TestFailed

import nose
from numpy import array, ndarray

from mayavi.tools.data_wizards.csv_sniff import \
     Sniff, loadtxt, loadtxt_unknown, array2dict

class Util(object):

    def assertNamedClose(self, x, y):
        self.assertEqual(x.shape, y.shape)
        self.assertEqual(x.dtype.names, y.dtype.names)
        for name in x.dtype.names:
            self.assertAllClose(x[name], y[name])

    def assertAllClose(self, x, y):
        self.assertEqual(len(x), len(y))
        for a, b in zip(x, y):
            self.assertClose(a, b)

    def assertClose(self, a, b):
        if isinstance(a, (int, float)):
            if repr(a) == 'nan':
                self.assert_(repr(b) == 'nan')
            else:
                self.assert_(abs(a - b) < 1e-6 * max(1, abs(a)),
                             '%r != %r  %r' % (a, b, abs(a - b)))

        elif isinstance(a, str):
            self.assertEqual(a, b)

        else:
            raise TestFailed("Hmm, did not expect: %r" % a)

class Test(unittest.TestCase, Util):

    def test_API(self):
        fo = open(TESTFN, 'wb')
        fo.write(''' "A", "B", "C"
                     1, 2, 3.2
                     7, 4, 1.87''')
        fo.close()

        s = Sniff(TESTFN)
        self.assertEqual(s.comments(), '#')
        self.assertEqual(s.delimiter(), ',')
        self.assertEqual(s.skiprows(), 1)
        self.assertEqual(s.dtype(), {'names': ('A', 'B', 'C'),
                                     'formats': (float, float, float)})
        x = s.loadtxt()
        y = array([(1.0, 2.0, 3.20),
                   (7.0, 4.0, 1.87)],
                  dtype=[('A', float), ('B', float), ('C', float)])
        self.assertNamedClose(x, y)

        y = loadtxt(TESTFN, **s.kwds())
        self.assertNamedClose(x, y)

        y = loadtxt_unknown(TESTFN)
        self.assertNamedClose(x, y)

        d = array2dict(y)
        self.assertEqual(type(d), type({}))
        self.assertAllClose(x['A'], [1, 7])
        self.assertAllClose(x['B'], [2, 4])
        self.assertAllClose(x['C'], [3.2, 1.87])

    def test_comment(self):
        fo = open(TESTFN, 'wb')
        fo.write('''
        % "A"  "B"  "C"
           1    2   4.2   % comment''')
        fo.close()

        s = Sniff(TESTFN)
        self.assertEqual(s.kwds(),
          {'dtype': {'names': ('A', 'B', 'C'),
                     'formats': (float, float, float)},
           'delimiter': None,
           'skiprows': 0,   # FIXME
           'comments': '%'})

    def test_tabs(self):
        fo = open(TESTFN, 'wb')
        fo.write('''54\t87\n21\t32''')
        fo.close()

        s = Sniff(TESTFN)
        self.assertEqual(s.delimiter(), None)
        self.assertEqual(s.skiprows(), 0)

    def test_nohead(self):
        fo = open(TESTFN, 'wb')
        fo.write('''Hello;54;87\nWorld;42;86.5''')
        fo.close()

        s = Sniff(TESTFN)
        self.assertEqual(s.kwds(),
          {'comments': '#',
           'delimiter': ';',
           'skiprows': 0,
           'dtype': {'names': ('Column 1', 'Column 2', 'Column 3'),
                     'formats': ('S5', float, float)}})

    def test_empty_file(self):
        fo = open(TESTFN, 'wb')
        fo.write('')
        fo.close()

        self.assertRaises(IndexError, Sniff, TESTFN)

    def tearDown(self):
        os.unlink(TESTFN)

class Test_csv_py_files(unittest.TestCase, Util):
    """
        These tests require files in csv_files/
    """
    def check(self, name, skip_if_win=False):
        """
            Check if the output array from csv_files/<name>.csv
            (which is of unkown format)
            is the same as the array in csv_files/<name>.py
        """
        if skip_if_win and sys.platform.startswith('win'):
            raise nose.SkipTest

        # Note: The files needed for the test are currently accessed directly.
        #       This assumes that the files are present, and not in a zipped
        #       egg.  traits.util.resource as supposed to always work, but
        #       when it ran the striped the tests in the EPD installer tests,
        #       it was broken.  Since we define zip_safe = False in setup.py
        #       it is safe to assume the files are always present.

        s = Sniff(os.path.join(os.path.dirname(__file__),
                               'csv_files', name + '.csv'))

        f_py = os.path.join(os.path.dirname(__file__),
                            'csv_files', name + '.py')

        if not sys.platform.startswith('win'):
            nan = float('nan') # must be in namespace for some .py files

        d = eval(open(f_py).read())

        self.assertEqual(d['kwds'], s.kwds())
        self.assertNamedClose(d['array'], s.loadtxt())

    def test_11(self):
        self.check('11')

    def test_1col(self):
        self.check('1col')

    def test_54(self):
        self.check('54')

    def test_79(self):
        self.check('79')

    def test_82(self):
        self.check('82')

    def test_89(self):
        self.check('89')

    def test_99(self):
        self.check('99')

    def test_colors(self):
        self.check('colors')

    def test_example1(self):
        # this test need to be skipped on Windows because the data file
        # has some floats nan, and float('nan') fails on on Windows.
        self.check('example1', skip_if_win=True)

    def test_hp11c(self):
        self.check('hp11c')

    def test_loc(self):
        self.check('loc')

    def test_multi_col(self):
        self.check('multi-col')

    def test_mydata(self):
        self.check('mydata')

    def test_OObeta3(self):
        self.check('OObeta3')

    def test_post(self):
        self.check('post')

    def test_webaccess(self):
        self.check('webaccess')

if __name__ == '__main__':
    unittest.main()
<commit_msg>Remove test_support module usage<commit_after>"""
Tests for the CSV file sniffer
"""
# Author: Ilan Schnell <ischnell@enthought.com>
# Copyright (c) 2008, Enthought, Inc.
# License: BSD Style.
import glob
import os
import os.path
import sys
import unittest
import tempfile

import nose
from numpy import array, ndarray

from mayavi.tools.data_wizards.csv_sniff import \
     Sniff, loadtxt, loadtxt_unknown, array2dict

class Util(unittest.TestCase):

    def assertNamedClose(self, x, y):
        self.assertEqual(x.shape, y.shape)
        self.assertEqual(x.dtype.names, y.dtype.names)
        for name in x.dtype.names:
            self.assertAllClose(x[name], y[name])

    def assertAllClose(self, x, y):
        self.assertEqual(len(x), len(y))
        for a, b in zip(x, y):
            self.assertClose(a, b)

    def assertClose(self, a, b):
        if isinstance(a, (int, float)):
            if repr(a) == 'nan':
                self.assert_(repr(b) == 'nan')
            else:
                self.assert_(abs(a - b) < 1e-6 * max(1, abs(a)),
                             '%r != %r  %r' % (a, b, abs(a - b)))

        elif isinstance(a, str):
            self.assertEqual(a, b)

        else:
            self.assertFalse("Hmm, did not expect: %s" % a)

class Test(Util):

    def test_API(self):
        fo = tempfile.mktemp()
        with open(fo, 'wb') as f:
            f.write(''' "A", "B", "C"
                         1, 2, 3.2
                         7, 4, 1.87''')

        s = Sniff(fo)
        self.assertEqual(s.comments(), '#')
        self.assertEqual(s.delimiter(), ',')
        self.assertEqual(s.skiprows(), 1)
        self.assertEqual(s.dtype(), {'names': ('A', 'B', 'C'),
                                     'formats': (float, float, float)})
        x = s.loadtxt()
        y = array([(1.0, 2.0, 3.20),
                   (7.0, 4.0, 1.87)],
                  dtype=[('A', float), ('B', float), ('C', float)])
        self.assertNamedClose(x, y)

        y = loadtxt(fo, **s.kwds())
        self.assertNamedClose(x, y)

        y = loadtxt_unknown(fo)
        self.assertNamedClose(x, y)

        d = array2dict(y)
        self.assertEqual(type(d), type({}))
        self.assertAllClose(x['A'], [1, 7])
        self.assertAllClose(x['B'], [2, 4])
        self.assertAllClose(x['C'], [3.2, 1.87])

    def test_comment(self):
        fo = tempfile.mktemp()
        with open(fo, 'wb') as f:
            f.write('''
            % "A"  "B"  "C"
               1    2   4.2   % comment''')

        s = Sniff(fo)
        self.assertEqual(s.kwds(),
          {'dtype': {'names': ('A', 'B', 'C'),
                     'formats': (float, float, float)},
           'delimiter': None,
           'skiprows': 0,   # FIXME
           'comments': '%'})

    def test_tabs(self):
        fo = tempfile.mktemp()
        with open(fo, 'wb') as f:
            f.write('''54\t87\n21\t32''')

        s = Sniff(fo)
        self.assertEqual(s.delimiter(), None)
        self.assertEqual(s.skiprows(), 0)

    def test_nohead(self):
        fo = tempfile.mktemp()
        with open(fo, 'wb') as f:
            f.write('''Hello;54;87\nWorld;42;86.5''')

        s = Sniff(fo)
        self.assertEqual(s.kwds(),
          {'comments': '#',
           'delimiter': ';',
           'skiprows': 0,
           'dtype': {'names': ('Column 1', 'Column 2', 'Column 3'),
                     'formats': ('S5', float, float)}})

    def test_empty_file(self):
        fo = tempfile.mktemp()
        with open(fo, 'wb') as f:
            f.write('')
        self.assertRaises(IndexError, Sniff, fo)

class Test_csv_py_files(Util):
    """
        These tests require files in csv_files/
    """
    def check(self, name, skip_if_win=False):
        """
            Check if the output array from csv_files/<name>.csv
            (which is of unkown format)
            is the same as the array in csv_files/<name>.py
        """
        if skip_if_win and sys.platform.startswith('win'):
            raise nose.SkipTest

        # Note: The files needed for the test are currently accessed directly.
        #       This assumes that the files are present, and not in a zipped
        #       egg.  traits.util.resource as supposed to always work, but
        #       when it ran the striped the tests in the EPD installer tests,
        #       it was broken.  Since we define zip_safe = False in setup.py
        #       it is safe to assume the files are always present.

        s = Sniff(os.path.join(os.path.dirname(__file__),
                               'csv_files', name + '.csv'))

        f_py = os.path.join(os.path.dirname(__file__),
                            'csv_files', name + '.py')

        if not sys.platform.startswith('win'):
            nan = float('nan') # must be in namespace for some .py files

        d = eval(open(f_py).read())

        self.assertEqual(d['kwds'], s.kwds())
        self.assertNamedClose(d['array'], s.loadtxt())

    def test_11(self):
        self.check('11')

    def test_1col(self):
        self.check('1col')

    def test_54(self):
        self.check('54')

    def test_79(self):
        self.check('79')

    def test_82(self):
        self.check('82')

    def test_89(self):
        self.check('89')

    def test_99(self):
        self.check('99')

    def test_colors(self):
        self.check('colors')

    def test_example1(self):
        # this test need to be skipped on Windows because the data file
        # has some floats nan, and float('nan') fails on on Windows.
        self.check('example1', skip_if_win=True)

    def test_hp11c(self):
        self.check('hp11c')

    def test_loc(self):
        self.check('loc')

    def test_multi_col(self):
        self.check('multi-col')

    def test_mydata(self):
        self.check('mydata')

    def test_OObeta3(self):
        self.check('OObeta3')

    def test_post(self):
        self.check('post')

    def test_webaccess(self):
        self.check('webaccess')

if __name__ == '__main__':
    unittest.main()
<|endoftext|>