<commit_before>"""
This module manages all market related activities
"""

import json
import logging
import hashlib
import random
from threading import Thread

from protocol import proto_page, query_page
from reputation import Reputation
from orders import Orders
import protocol
import lookup
from pymongo import MongoClient
from data_uri import DataURI
from PIL import Image, ImageOps
from StringIO import StringIO
import base64

class Market(object):

    def __init__(self, transport):

        """This class manages the active market for the application

        Attributes:
          _transport (CryptoTransportLayer): Transport layer for messaging between nodes.
          _dht (DHT): For storage across the network.
          _market_id (int): Indicates which local market we're working with.

        """

        # Current
        self._transport = transport
        self._dht = transport.getDHT()
        self._market_id = transport.getMarketID()
        self._myself = transport.getMyself()
        self._peers = self._dht.getActivePeers()

        # Legacy for now
        self.query_ident = None
        self.reputation = Reputation(self._transport)
        self.orders = Orders(self._transport)
        self.order_entries = self.orders._orders
        self.nicks = {}
        self.pages = {}
        self.welcome = False
        self.mypage = None
        self.signature = None
        self.nickname = None

        self._log = logging.getLogger('[%s] %s' % (self._market_id, self.__class__.__name__))
        self._log.info("Loading Market %s" % self._market_id)

        #MONGODB_URI = 'mongodb://localhost:27017'
        _dbclient = MongoClient()
        self._db = _dbclient.openbazaar

        self.settings = self._transport.settings

        welcome = True

        if self.settings:
            if  'welcome' in self.settings.keys() and self.settings['welcome']:
                welcome = False

        # Register callbacks for incoming events
        self._transport.add_callback('query_myorders', self.on_query_myorders)
        self._transport.add_callback('peer', self.on_peer)
        self._transport.add_callback('query_page', self.on_query_page)
        self._transport.add_callback('page', self.on_page)
        self._transport.add_callback('negotiate_pubkey', self.on_negotiate_pubkey)
        self._transport.add_callback('proto_response_pubkey', self.on_response_pubkey)

        self.load_page(welcome)

    def load_page(self, welcome):

        nickname = self.settings['nickname'] if self.settings.has_key("nickname") else ""
        store_description = self.settings['storeDescription'] if self.settings.has_key("storeDescription") else ""

        tagline = "%s: %s" % (nickname, store_description)
        self.mypage = tagline
        self.nickname = nickname
        self.signature = self._transport._myself.sign(tagline)

        if welcome:
            self._db.settings.update({}, {"$set":{"welcome":"noshow"}})
        else:
            self.welcome = False

    def save_product(self, msg):
        #self._log.debug("Saving product: %s" % msg)

        msg['market_id'] = self._market_id

        product_id = msg['id'] if msg.has_key("id") else ""

        if product_id == "":
            product_id = random.randint(0, 1000000)

        if not msg.has_key("productPrice") or not msg['productPrice'] > 0:
            msg['productPrice'] = 0

        if not msg.has_key("productQuantity") or not msg['productQuantity'] > 0:
            msg['productQuantity'] = 1

        if msg.has_key('productImageData'):

            uri = DataURI(msg['productImageData'])
            imageData = uri.data
            mime_type = uri.mimetype
            charset = uri.charset

            image = Image.open(StringIO(imageData))
            croppedImage = ImageOps.fit(image, (100, 100), centering=(0.5, 0.5))
            data = StringIO()
            croppedImage.save(data, format='PNG')

            new_uri = DataURI.make('image/png', charset=charset, base64=True, data=data.getvalue())
            data.close()
            msg['productImageData'] = new_uri

        # Save product listing to DHT
        listing = json.dumps(msg)
        listing_key = hashlib.sha1(listing).hexdigest()

        hash_value = hashlib.new('ripemd160')
        hash_value.update(listing_key)
        listing_key = hash_value.hexdigest()

        msg['key'] = listing_key

        self._db.products.update({'id':product_id}, {'$set':msg}, True)

        self._log.debug('New Listing Key: %s' % listing_key)

        # Store listing

        self._transport._dht.iterativeStore(self._transport, listing_key, listing, self._transport._guid)

        self.update_listings_index()

        # If keywords store them in the keyword index

    def republish_listing(self, msg):

        listing_id = msg.get('productID')
        listing = self._db.products.find_one({'id':listing_id})
        key = listing['key']

        listing = json.loads(listing)
        print listing

        self._transport._dht.iterativeStore(self._transport, key, listing, self._transport._guid)

    def update_listings_index(self):

        # Store to marketplace listing index
        listing_index_key = hashlib.sha1('listings-%s' % self._transport._guid).hexdigest()
        hashvalue = hashlib.new('ripemd160')
        hashvalue.update(listing_index_key)
        listing_index_key = hashvalue.hexdigest()

        # Calculate index of listings
        listing_ids = self._db.products.find({'market_id':self._transport._market_id}, {'key':1})
        my_listings = []
        for listing_id in listing_ids:
            my_listings.append(listing_id['key'])

        self._log.debug('My Listings: %s' % my_listings)

        # Sign listing index for validation and tamper resistance
        data_string = str({'guid':self._transport._guid, 'listings': my_listings})
        signature = self._myself.sign(data_string).encode('hex')

        value = {'signature': signature, 'data': {'guid':self._transport._guid, 'listings': my_listings}}

        # Pass off to thread to keep GUI snappy
        Thread(target=self._transport._dht.iterativeStore, args=(self._transport, listing_index_key, value, self._transport._guid,)).start()

    def remove_product(self, msg):
        self._log.info("Removing product: %s" % msg)
        self._db.products.remove({'id':msg['productID']})
        self.update_listings_index()

    def get_products(self):
        self._log.info('Getting products for market: %s' % self._transport._market_id)
        products = self._db.products.find({'market_id':self._transport._market_id})
        my_products = []

        for product in products:
            my_products.append({"productTitle":product['productTitle'] if product.has_key("productTitle") else "",
                                "id":product['id'] if product.has_key("id") else "",
                                "productDescription":product['productDescription'] if product.has_key("productDescription") else "",
                                "productPrice":product['productPrice'] if product.has_key("productPrice") else "",
                                "productShippingPrice":product['productShippingPrice'] if product.has_key("productShippingPrice") else "",
                                "productTags":product['productTags'] if product.has_key("productTags") else "",
                                "productImageData":product['productImageData'] if product.has_key("productImageData") else "",
                                "productQuantity":product['productQuantity'] if product.has_key("productQuantity") else "",
                                "key":product['key'] if product.has_key("key") else "",
                               })

        return {"products": my_products}

    # SETTINGS

    def save_settings(self, msg):
        self._log.info("Settings to save %s" % msg)
        self._log.info(self._transport)
        self._db.settings.update({'id':'%s'%self._transport._market_id}, {'$set':msg}, True)

    def get_settings(self):
        self._log.info(self._transport._market_id)
        settings = self._db.settings.find_one({'id':'%s'%self._transport._market_id})

        if settings:
            return {"bitmessage": settings['bitmessage'] if settings.has_key("bitmessage") else "",
                    "email": settings['email'] if settings.has_key("email") else "",
                    "PGPPubKey": settings['PGPPubKey'] if settings.has_key("PGPPubKey") else "",
                    "pubkey": settings['pubkey'] if settings.has_key("pubkey") else "",
                    "nickname": settings['nickname'] if settings.has_key("nickname") else "",
                    "secret": settings['secret'] if settings.has_key("secret") else "",
                    "welcome": settings['welcome'] if settings.has_key("welcome") else "",
                    "escrowAddresses": settings['escrowAddresses'] if settings.has_key("escrowAddresses") else "",
                    "storeDescription": settings['storeDescription'] if settings.has_key("storeDescription") else "",
                    "city": settings['city'] if settings.has_key("city") else "",
                    "stateRegion": settings['stateRegion'] if settings.has_key("stateRegion") else "",
                    "street1": settings['street1'] if settings.has_key("street1") else "",
                    "street2": settings['street2'] if settings.has_key("street2") else "",
                    "countryCode": settings['countryCode'] if settings.has_key("countryCode") else "",
                    "zip": settings['zip'] if settings.has_key("zip") else "",
                    "arbiterDescription": settings['arbiterDescription'] if settings.has_key("arbiterDescription") else "",
                    "arbiter": settings['arbiter'] if settings.has_key("arbiter") else "",
                   }

    # PAGE QUERYING

    def query_page(self, find_guid):

        self._log.info('Querying page: %s' % find_guid)
        msg = query_page(find_guid)
        msg['uri'] = self._transport._uri
        msg['senderGUID'] = self._transport.guid
        msg['pubkey'] = self._transport.pubkey

        self._transport.send(msg, find_guid)

    def on_page(self, page):

        self._log.info("Page received and being stored in Market")

        #pubkey = page.get('pubkey')
        guid = page.get('senderGUID')
        page = page.get('text')

        if guid and page:
            self.pages[guid] = page

    # Return your page info if someone requests it on the network
    def on_query_page(self, peer):
        self._log.info("Someone is querying for your page")
        self.settings = self.get_settings()
        #self._log.info(base64.b64encode(self.settings['storeDescription']))
        self._transport.send(proto_page(self._transport._uri,
                                        self._transport.pubkey,
                                        self._transport.guid,
                                        self.settings['storeDescription'],
                                        self.signature,
                                        self.settings['nickname'],
                                        self.settings['PGPPubKey'] if self.settings.has_key('PGPPubKey') else '',
                                        self.settings['email'] if self.settings.has_key('email') else '',
                                        self.settings['bitmessage'] if self.settings.has_key('bitmessage') else ''),
                                        peer['senderGUID']
                                        )

    def on_query_myorders(self, peer):
        self._log.info("Someone is querying for your page: %s" % peer)

    def on_peer(self, peer):
        pass

    def on_negotiate_pubkey(self, ident_pubkey):
        self._log.info("Someone is asking for your real pubKey")
        assert "nickname" in ident_pubkey
        assert "ident_pubkey" in ident_pubkey
        nickname = ident_pubkey['nickname']
        ident_pubkey = ident_pubkey['ident_pubkey'].decode("hex")
        self._transport.respond_pubkey_if_mine(nickname, ident_pubkey)

    def on_response_pubkey(self, response):
        self._log.info("got a pubkey!")
        assert "pubkey" in response
        assert "nickname" in response
        assert "signature" in response
        pubkey = response["pubkey"].decode("hex")
        #signature = response["signature"].decode("hex")
        nickname = response["nickname"]
        # Cache mapping for later.
        if nickname not in self._transport.nick_mapping:
            self._transport.nick_mapping[nickname] = [None, pubkey]
        # Verify signature here...
        # Add to our dict.
        self._transport.nick_mapping[nickname][1] = pubkey
        self._log.info("[market] mappings: ###############")
        for key, value in self._transport.nick_mapping.iteritems():
            self._log.info("'%s' -> '%s' (%s)" % (
                key, value[1].encode("hex") if value[1] is not None else value[1],
                value[0].encode("hex") if value[0] is not None else value[0]))
        self._log.info("##################################")

    def lookup(self, msg):

        if self.query_ident is None:
            self._log.info("Initializing identity query")
            self.query_ident = lookup.QueryIdent()

        nickname = str(msg["text"])
        key = self.query_ident.lookup(nickname)
        if key is None:
            self._log.info("Key not found for this nickname")
            return "Key not found for this nickname", None

        self._log.info("Found key: %s " % key.encode("hex"))
        if nickname in self._transport.nick_mapping:
            self._log.info("Already have a cached mapping, just adding key there.")
            response = {'nickname': nickname,
                        'pubkey': self._transport.nick_mapping[nickname][1].encode('hex'),
                        'signature': self._transport.nick_mapping[nickname][0].encode('hex'),
                        'type': 'response_pubkey',
                       }
            self._transport.nick_mapping[nickname][0] = key
            return None, response

        self._transport.nick_mapping[nickname] = [key, None]

        self._transport.send(protocol.negotiate_pubkey(nickname, key))
<commit_msg>Fixing bug with Republish<commit_after>"""
This module manages all market related activities
"""

import json
import logging
import hashlib
import random
from threading import Thread

from protocol import proto_page, query_page
from reputation import Reputation
from orders import Orders
import protocol
import lookup
from pymongo import MongoClient
from data_uri import DataURI
from PIL import Image, ImageOps
from StringIO import StringIO
import base64

class Market(object):

    def __init__(self, transport):

        """This class manages the active market for the application

        Attributes:
          _transport (CryptoTransportLayer): Transport layer for messaging between nodes.
          _dht (DHT): For storage across the network.
          _market_id (int): Indicates which local market we're working with.

        """

        # Current
        self._transport = transport
        self._dht = transport.getDHT()
        self._market_id = transport.getMarketID()
        self._myself = transport.getMyself()
        self._peers = self._dht.getActivePeers()

        # Legacy for now
        self.query_ident = None
        self.reputation = Reputation(self._transport)
        self.orders = Orders(self._transport)
        self.order_entries = self.orders._orders
        self.nicks = {}
        self.pages = {}
        self.welcome = False
        self.mypage = None
        self.signature = None
        self.nickname = None

        self._log = logging.getLogger('[%s] %s' % (self._market_id, self.__class__.__name__))
        self._log.info("Loading Market %s" % self._market_id)

        #MONGODB_URI = 'mongodb://localhost:27017'
        _dbclient = MongoClient()
        self._db = _dbclient.openbazaar

        self.settings = self._transport.settings

        welcome = True

        if self.settings:
            if  'welcome' in self.settings.keys() and self.settings['welcome']:
                welcome = False

        # Register callbacks for incoming events
        self._transport.add_callback('query_myorders', self.on_query_myorders)
        self._transport.add_callback('peer', self.on_peer)
        self._transport.add_callback('query_page', self.on_query_page)
        self._transport.add_callback('page', self.on_page)
        self._transport.add_callback('negotiate_pubkey', self.on_negotiate_pubkey)
        self._transport.add_callback('proto_response_pubkey', self.on_response_pubkey)

        self.load_page(welcome)

    def load_page(self, welcome):

        nickname = self.settings['nickname'] if self.settings.has_key("nickname") else ""
        store_description = self.settings['storeDescription'] if self.settings.has_key("storeDescription") else ""

        tagline = "%s: %s" % (nickname, store_description)
        self.mypage = tagline
        self.nickname = nickname
        self.signature = self._transport._myself.sign(tagline)

        if welcome:
            self._db.settings.update({}, {"$set":{"welcome":"noshow"}})
        else:
            self.welcome = False

    def save_product(self, msg):
        #self._log.debug("Saving product: %s" % msg)

        msg['market_id'] = self._market_id

        product_id = msg['id'] if msg.has_key("id") else ""

        if product_id == "":
            product_id = random.randint(0, 1000000)

        if not msg.has_key("productPrice") or not msg['productPrice'] > 0:
            msg['productPrice'] = 0

        if not msg.has_key("productQuantity") or not msg['productQuantity'] > 0:
            msg['productQuantity'] = 1

        if msg.has_key('productImageData'):

            uri = DataURI(msg['productImageData'])
            imageData = uri.data
            mime_type = uri.mimetype
            charset = uri.charset

            image = Image.open(StringIO(imageData))
            croppedImage = ImageOps.fit(image, (100, 100), centering=(0.5, 0.5))
            data = StringIO()
            croppedImage.save(data, format='PNG')

            new_uri = DataURI.make('image/png', charset=charset, base64=True, data=data.getvalue())
            data.close()
            msg['productImageData'] = new_uri

        # Save product listing to DHT
        listing = json.dumps(msg)
        listing_key = hashlib.sha1(listing).hexdigest()

        hash_value = hashlib.new('ripemd160')
        hash_value.update(listing_key)
        listing_key = hash_value.hexdigest()

        msg['key'] = listing_key

        self._db.products.update({'id':product_id}, {'$set':msg}, True)

        self._log.debug('New Listing Key: %s' % listing_key)

        # Store listing

        self._transport._dht.iterativeStore(self._transport, listing_key, listing, self._transport._guid)

        self.update_listings_index()

        # If keywords store them in the keyword index

    def republish_listing(self, msg):

        listing_id = msg.get('productID')
        listing = self._db.products.find_one({'id':listing_id})
        key = listing['key']

        #listing = json.loads(listing)
        print listing

        self._transport._dht.iterativeStore(self._transport, key, listing, self._transport._guid)

    def update_listings_index(self):

        # Store to marketplace listing index
        listing_index_key = hashlib.sha1('listings-%s' % self._transport._guid).hexdigest()
        hashvalue = hashlib.new('ripemd160')
        hashvalue.update(listing_index_key)
        listing_index_key = hashvalue.hexdigest()

        # Calculate index of listings
        listing_ids = self._db.products.find({'market_id':self._transport._market_id}, {'key':1})
        my_listings = []
        for listing_id in listing_ids:
            my_listings.append(listing_id['key'])

        self._log.debug('My Listings: %s' % my_listings)

        # Sign listing index for validation and tamper resistance
        data_string = str({'guid':self._transport._guid, 'listings': my_listings})
        signature = self._myself.sign(data_string).encode('hex')

        value = {'signature': signature, 'data': {'guid':self._transport._guid, 'listings': my_listings}}

        # Pass off to thread to keep GUI snappy
        Thread(target=self._transport._dht.iterativeStore, args=(self._transport, listing_index_key, value, self._transport._guid,)).start()

    def remove_product(self, msg):
        self._log.info("Removing product: %s" % msg)
        self._db.products.remove({'id':msg['productID']})
        self.update_listings_index()

    def get_products(self):
        self._log.info('Getting products for market: %s' % self._transport._market_id)
        products = self._db.products.find({'market_id':self._transport._market_id})
        my_products = []

        for product in products:
            my_products.append({"productTitle":product['productTitle'] if product.has_key("productTitle") else "",
                                "id":product['id'] if product.has_key("id") else "",
                                "productDescription":product['productDescription'] if product.has_key("productDescription") else "",
                                "productPrice":product['productPrice'] if product.has_key("productPrice") else "",
                                "productShippingPrice":product['productShippingPrice'] if product.has_key("productShippingPrice") else "",
                                "productTags":product['productTags'] if product.has_key("productTags") else "",
                                "productImageData":product['productImageData'] if product.has_key("productImageData") else "",
                                "productQuantity":product['productQuantity'] if product.has_key("productQuantity") else "",
                                "key":product['key'] if product.has_key("key") else "",
                               })

        return {"products": my_products}

    # SETTINGS

    def save_settings(self, msg):
        self._log.info("Settings to save %s" % msg)
        self._log.info(self._transport)
        self._db.settings.update({'id':'%s'%self._transport._market_id}, {'$set':msg}, True)

    def get_settings(self):
        self._log.info(self._transport._market_id)
        settings = self._db.settings.find_one({'id':'%s'%self._transport._market_id})

        if settings:
            return {"bitmessage": settings['bitmessage'] if settings.has_key("bitmessage") else "",
                    "email": settings['email'] if settings.has_key("email") else "",
                    "PGPPubKey": settings['PGPPubKey'] if settings.has_key("PGPPubKey") else "",
                    "pubkey": settings['pubkey'] if settings.has_key("pubkey") else "",
                    "nickname": settings['nickname'] if settings.has_key("nickname") else "",
                    "secret": settings['secret'] if settings.has_key("secret") else "",
                    "welcome": settings['welcome'] if settings.has_key("welcome") else "",
                    "escrowAddresses": settings['escrowAddresses'] if settings.has_key("escrowAddresses") else "",
                    "storeDescription": settings['storeDescription'] if settings.has_key("storeDescription") else "",
                    "city": settings['city'] if settings.has_key("city") else "",
                    "stateRegion": settings['stateRegion'] if settings.has_key("stateRegion") else "",
                    "street1": settings['street1'] if settings.has_key("street1") else "",
                    "street2": settings['street2'] if settings.has_key("street2") else "",
                    "countryCode": settings['countryCode'] if settings.has_key("countryCode") else "",
                    "zip": settings['zip'] if settings.has_key("zip") else "",
                    "arbiterDescription": settings['arbiterDescription'] if settings.has_key("arbiterDescription") else "",
                    "arbiter": settings['arbiter'] if settings.has_key("arbiter") else "",
                   }

    # PAGE QUERYING

    def query_page(self, find_guid):

        self._log.info('Querying page: %s' % find_guid)
        msg = query_page(find_guid)
        msg['uri'] = self._transport._uri
        msg['senderGUID'] = self._transport.guid
        msg['pubkey'] = self._transport.pubkey

        self._transport.send(msg, find_guid)

    def on_page(self, page):

        self._log.info("Page received and being stored in Market")

        #pubkey = page.get('pubkey')
        guid = page.get('senderGUID')
        page = page.get('text')

        if guid and page:
            self.pages[guid] = page

    # Return your page info if someone requests it on the network
    def on_query_page(self, peer):
        self._log.info("Someone is querying for your page")
        self.settings = self.get_settings()
        #self._log.info(base64.b64encode(self.settings['storeDescription']))
        self._transport.send(proto_page(self._transport._uri,
                                        self._transport.pubkey,
                                        self._transport.guid,
                                        self.settings['storeDescription'],
                                        self.signature,
                                        self.settings['nickname'],
                                        self.settings['PGPPubKey'] if self.settings.has_key('PGPPubKey') else '',
                                        self.settings['email'] if self.settings.has_key('email') else '',
                                        self.settings['bitmessage'] if self.settings.has_key('bitmessage') else ''),
                                        peer['senderGUID']
                                        )

    def on_query_myorders(self, peer):
        self._log.info("Someone is querying for your page: %s" % peer)

    def on_peer(self, peer):
        pass

    def on_negotiate_pubkey(self, ident_pubkey):
        self._log.info("Someone is asking for your real pubKey")
        assert "nickname" in ident_pubkey
        assert "ident_pubkey" in ident_pubkey
        nickname = ident_pubkey['nickname']
        ident_pubkey = ident_pubkey['ident_pubkey'].decode("hex")
        self._transport.respond_pubkey_if_mine(nickname, ident_pubkey)

    def on_response_pubkey(self, response):
        self._log.info("got a pubkey!")
        assert "pubkey" in response
        assert "nickname" in response
        assert "signature" in response
        pubkey = response["pubkey"].decode("hex")
        #signature = response["signature"].decode("hex")
        nickname = response["nickname"]
        # Cache mapping for later.
        if nickname not in self._transport.nick_mapping:
            self._transport.nick_mapping[nickname] = [None, pubkey]
        # Verify signature here...
        # Add to our dict.
        self._transport.nick_mapping[nickname][1] = pubkey
        self._log.info("[market] mappings: ###############")
        for key, value in self._transport.nick_mapping.iteritems():
            self._log.info("'%s' -> '%s' (%s)" % (
                key, value[1].encode("hex") if value[1] is not None else value[1],
                value[0].encode("hex") if value[0] is not None else value[0]))
        self._log.info("##################################")

    def lookup(self, msg):

        if self.query_ident is None:
            self._log.info("Initializing identity query")
            self.query_ident = lookup.QueryIdent()

        nickname = str(msg["text"])
        key = self.query_ident.lookup(nickname)
        if key is None:
            self._log.info("Key not found for this nickname")
            return "Key not found for this nickname", None

        self._log.info("Found key: %s " % key.encode("hex"))
        if nickname in self._transport.nick_mapping:
            self._log.info("Already have a cached mapping, just adding key there.")
            response = {'nickname': nickname,
                        'pubkey': self._transport.nick_mapping[nickname][1].encode('hex'),
                        'signature': self._transport.nick_mapping[nickname][0].encode('hex'),
                        'type': 'response_pubkey',
                       }
            self._transport.nick_mapping[nickname][0] = key
            return None, response

        self._transport.nick_mapping[nickname] = [key, None]

        self._transport.send(protocol.negotiate_pubkey(nickname, key))
<|endoftext|>


<commit_before>from collections import defaultdict, Counter
import csv
import functools
import multiprocessing
import os

import axelrod as axl
#from approximate_moran import *

def write_csv(outcomes, filename="outcomes.csv", append=False):
    s = 'w'
    if append:
        s = 'a'
    writer = csv.writer(open(filename, s))
    for pair, counter in outcomes.items():
        for scores, count in counter.items():
            row = [pair[0], pair[1], scores[0], scores[1], count]
            writer.writerow(row)

def read_csv(filename="outcomes.csv"):
    reader = csv.reader(open(filename))
    outcomes = defaultdict(Counter)
    for row in reader:
        p1, p2, s1, s2, count = row
        outcomes[(p1, p2)][(float(s1), float(s2))] = int(count)
        outcomes[(p2, p1)][(float(s2), float(s1))] = int(count)
    return outcomes

def generate_matchups(players):
    # Want the triangular product
    for i in range(len(players)):
        for j in range(i, len(players)):
            if i == j:
                yield players[i].clone(), players[j]
            else:
                yield players[i], players[j]

def generate_matchups_indices(num_players):
    # Want the triangular product
    for i in range(num_players):
        for j in range(i, num_players):
            yield i, j

def sample_match_outcomes(players, turns, repetitions, noise=0):
    """
    Play all matches between pairs of players and return a dictionary mapping
    player names to outcomes.
    """
    match_outcomes = dict()
    matchups = list(generate_matchups(players))

    for pairs in matchups:

        match = axl.Match(pairs, turns=turns, noise=noise)
        rs = repetitions
        if not match._stochastic:
            rs = 1
        outcomes = []
        for _ in range(rs):
            match.play()
            outcomes.append(match.final_score_per_turn())

        counts = Counter(outcomes)

        player_names = tuple(map(str, pairs))
        match_outcomes[player_names] = counts

    return match_outcomes

def write_winner(filename, turns, repetitions, noise, i, j, seed=None):
    """
    Write the winner of a Moran process to file
    """
    if seed:
        axl.seed(seed)  # Seed the process

    pairs = (players[i].clone(), players[j].clone())
    match = axl.Match(pairs, turns=turns, noise=noise)
    rs = repetitions
    if not match._stochastic and noise == 0:
        rs = 1
    outcomes = []
    for _ in range(rs):
        match.play()
        outcomes.append(match.final_score_per_turn())

    counts = Counter(outcomes)
    player_names = tuple(map(str, pairs))
    match_outcomes = dict()
    match_outcomes[player_names] = counts
    write_csv(match_outcomes, filename=filename, append=True)

def sample_match_outcomes_parallel(turns, repetitions, filename, noise=0,
                                   processes=None):
    """
    Parallel matches.
    """

    player_indices = range(len(players))
    if processes is None:
        for i in player_indices:
            print(i, len(players))
            for j in player_indices:
                for seed in range(repetitions):
                    write_winner(filename, turns, repetitions, noise, i, j,
                                 seed)
    else:
        func = functools.partial(write_winner, filename, turns, repetitions,
                                 noise)
        p = multiprocessing.Pool(processes)

        args = generate_matchups_indices(len(players))
        p.starmap(func, args)

if __name__ == "__main__":
    # players are global
    from players import selected_players
    players = selected_players()

    print(len(list(map(str, players))))

    try:
        os.remove("outcomes.csv")
        os.remove("outcomes_noise.csv")
    except FileNotFoundError:
        pass

    repetitions = 10000
    turns = 200

    sample_match_outcomes_parallel(turns=turns, repetitions=repetitions,
                                   filename="outcomes.csv", noise=0,
                                   processes=4)

    sample_match_outcomes_parallel(turns=turns, repetitions=repetitions,
                                   filename="outcomes_noise.csv", noise=0.05,
                                   processes=4)
<commit_msg>Add an exception.<commit_after>from collections import defaultdict, Counter
import csv
import functools
import multiprocessing
import os

import axelrod as axl
#from approximate_moran import *

def write_csv(outcomes, filename="outcomes.csv", append=False):
    s = 'w'
    if append:
        s = 'a'
    writer = csv.writer(open(filename, s))
    for pair, counter in outcomes.items():
        for scores, count in counter.items():
            row = [pair[0], pair[1], scores[0], scores[1], count]
            writer.writerow(row)

def read_csv(filename="outcomes.csv"):
    reader = csv.reader(open(filename))
    outcomes = defaultdict(Counter)
    for row in reader:
        p1, p2, s1, s2, count = row
        outcomes[(p1, p2)][(float(s1), float(s2))] = int(count)
        outcomes[(p2, p1)][(float(s2), float(s1))] = int(count)
    return outcomes

def generate_matchups(players):
    # Want the triangular product
    for i in range(len(players)):
        for j in range(i, len(players)):
            if i == j:
                yield players[i].clone(), players[j]
            else:
                yield players[i], players[j]

def generate_matchups_indices(num_players):
    # Want the triangular product
    for i in range(num_players):
        for j in range(i, num_players):
            yield i, j

def sample_match_outcomes(players, turns, repetitions, noise=0):
    """
    Play all matches between pairs of players and return a dictionary mapping
    player names to outcomes.
    """
    match_outcomes = dict()
    matchups = list(generate_matchups(players))

    for pairs in matchups:

        match = axl.Match(pairs, turns=turns, noise=noise)
        rs = repetitions
        if not match._stochastic:
            rs = 1
        outcomes = []
        for _ in range(rs):
            match.play()
            outcomes.append(match.final_score_per_turn())

        counts = Counter(outcomes)

        player_names = tuple(map(str, pairs))
        match_outcomes[player_names] = counts

    return match_outcomes

def write_winner(filename, turns, repetitions, noise, i, j, seed=None):
    """
    Write the outcome of a match to file.
    """
    if seed:
        axl.seed(seed)  # Seed the process

    pairs = (players[i].clone(), players[j].clone())
    match = axl.Match(pairs, turns=turns, noise=noise)
    rs = repetitions
    if not match._stochastic and noise == 0:
        rs = 1
    outcomes = []
    for _ in range(rs):
        try:
            match.play()
            outcomes.append(match.final_score_per_turn())
        except KeyError:
            print(turns, repetitions, noise, i, j, seed)

    counts = Counter(outcomes)
    player_names = tuple(map(str, pairs))
    match_outcomes = dict()
    match_outcomes[player_names] = counts
    write_csv(match_outcomes, filename=filename, append=True)

def sample_match_outcomes_parallel(turns, repetitions, filename, noise=0,
                                   processes=None):
    """
    Parallel matches.
    """

    player_indices = range(len(players))
    if processes is None:
        for i in player_indices:
            print(i, len(players))
            for j in player_indices:
                for seed in range(repetitions):
                    write_winner(filename, turns, repetitions, noise, i, j,
                                 seed)
    else:
        func = functools.partial(write_winner, filename, turns, repetitions,
                                 noise)
        p = multiprocessing.Pool(processes)

        args = generate_matchups_indices(len(players))
        p.starmap(func, args)

if __name__ == "__main__":
    # players are global
    from players import selected_players
    players = selected_players()

    print(len(list(map(str, players))))

    try:
        os.remove("outcomes.csv")
        os.remove("outcomes_noise.csv")
    except FileNotFoundError:
        pass

    repetitions = 200
    turns = 50

    sample_match_outcomes_parallel(turns=turns, repetitions=repetitions,
                                   filename="outcomes.csv", noise=0,
                                   processes=4)

    sample_match_outcomes_parallel(turns=turns, repetitions=repetitions,
                                   filename="outcomes_noise.csv", noise=0.05,
                                   processes=4)
<|endoftext|>


<commit_before>"""Constants and abstract classes."""
from __future__ import absolute_import, division, print_function, unicode_literals

import copy
import logging
import operator

import six

from .permissions import PermissionsManager

class BaseResource(object):
    """Abstract resource.

    One and only one of the identifiers (slug, id or model_data)
    should be given.

    :param resolwe: Resolwe instance
    :type resolwe: Resolwe object
    :param model_data: Resource model data

    """

    endpoint = None
    query_endpoint = None
    query_method = 'GET'

    WRITABLE_FIELDS = ()
    UPDATE_PROTECTED_FIELDS = ()
    READ_ONLY_FIELDS = ('id',)

    ALL_PERMISSIONS = []  # override this in subclass

    def __init__(self, resolwe, **model_data):
        """Verify that only a single attribute of slug, id or model_data given."""
        self._original_values = {}

        #: unique identifier
        self.id = None  # pylint: disable=invalid-name

        self.api = operator.attrgetter(self.endpoint)(resolwe.api)
        self.resolwe = resolwe

        if model_data:
            self._update_fields(model_data)

        self.logger = logging.getLogger(__name__)

    def fields(self):
        """Resource fields."""
        return self.WRITABLE_FIELDS + self.UPDATE_PROTECTED_FIELDS + self.READ_ONLY_FIELDS

    def _update_fields(self, payload):
        """Update fields of the local resource based on the server values.

        :param dict payload: Resource field values

        """
        self._original_values = copy.deepcopy(payload)
        for field_name in self.fields():
            setattr(self, field_name, payload.get(field_name, None))

    def update(self):
        """Update resource fields from the server."""
        response = self.api(self.id).get()
        self._update_fields(response)

    def _dehydrate_resources(self, obj):
        """Iterate through object and replace all objects with their ids."""
        # Prevent circular imports:
        from .descriptor import DescriptorSchema
        from .process import Process

        if isinstance(obj, DescriptorSchema) or isinstance(obj, Process):
            return obj.slug
        if isinstance(obj, BaseResource):
            return obj.id
        if isinstance(obj, list):
            return [self._dehydrate_resources(element) for element in obj]
        if isinstance(obj, dict):
            return {key: self._dehydrate_resources(value) for key, value in six.iteritems(obj)}
        return obj

    def save(self):
        """Save resource to the server."""
        def field_changed(field_name):
            """Check if local field value is different from the server."""
            return getattr(self, field_name) != self._original_values.get(field_name)

        def assert_fields_changed(field_names):
            """Check if local field value is different from the server."""
            changed_fields = [field_name for field_name in field_names if
                              field_changed(field_name)]

            if changed_fields:
                msg = "Not allowed to change read only fields {}".format(', '.join(changed_fields))
                raise ValueError(msg)

        if self.id:  # update resource
            assert_fields_changed(self.READ_ONLY_FIELDS + self.UPDATE_PROTECTED_FIELDS)

            payload = {}
            for field_name in self.WRITABLE_FIELDS:
                if field_changed(field_name):
                    payload[field_name] = self._dehydrate_resources(getattr(self, field_name))

            if payload:
                response = self.api(self.id).patch(payload)
                self._update_fields(response)

        else:  # create resource
            assert_fields_changed(self.READ_ONLY_FIELDS)

            field_names = self.WRITABLE_FIELDS + self.UPDATE_PROTECTED_FIELDS
            payload = {field_name: self._dehydrate_resources(getattr(self, field_name))
                       for field_name in field_names if getattr(self, field_name) is not None}

            response = self.api.post(payload)
            self._update_fields(response)

    def delete(self, force=False):
        """Delete the resource object from the server."""
        if force is not True:
            user_input = six.moves.input('Do you really want to delete {}?[yN] '.format(self))

            if user_input.strip().lower() != 'y':
                return

        self.api(self.id).delete()

    def __setattr__(self, name, value):
        """Detect changes of read only fields.

        This method detects changes of scalar fields and references. A
        more comprehensive check is called before save.

        """
        if (hasattr(self, '_original_values')
                and name in self._original_values
                and name in self.READ_ONLY_FIELDS
                and value != self._original_values[name]):
            raise ValueError("Can not change read only field {}".format(name))

        super(BaseResource, self).__setattr__(name, value)

    def __eq__(self, obj):
        """Evaluate if objects are the same."""
        if self.__class__ == obj.__class__ and self.id == obj.id:
            return True
        else:
            return False

class BaseResolweResource(BaseResource):
    """Base class for Resolwe resources.

    One and only one of the identifiers (slug, id or model_data)
    should be given.

    :param resolwe: Resolwe instance
    :type resolwe: Resolwe object
    :param model_data: Resource model data

    """

    _permissions = None

    WRITABLE_FIELDS = ('slug', 'name')
    UPDATE_PROTECTED_FIELDS = ('contributor', )
    READ_ONLY_FIELDS = ('id', 'version', 'created', 'modified', 'current_user_permissions')

    def __init__(self, resolwe, **model_data):
        """Initialize attributes."""
        #: a descriptive name of the resource
        self.name = None
        #: the resource version
        self.version = None
        #: human-readable unique identifier
        self.slug = None
        #: user id of the contributor
        self.contributor = None
        #: date of creation
        self.created = None
        #: date of latest modification
        self.modified = None
        #: permissions - (view/download/add/edit/share/owner for user/group/public)
        self.current_user_permissions = None

        BaseResource.__init__(self, resolwe, **model_data)

    @property
    def permissions(self):
        """TODO."""
        if not self.id:
            raise ValueError('Instance must be saved before accessing `permissions` attribute.')
        if not self._permissions:
            self._permissions = PermissionsManager(
                self.ALL_PERMISSIONS,
                self.api(self.id),
                self.resolwe
            )

        return self._permissions

    def update(self):
        """Clear permissions cache and update the object."""
        self.permissions.clear_cache()

        super(BaseResolweResource, self).update()

    def __repr__(self):
        """Format resource name."""
        rep = "{} <id: {}, slug: '{}', name: '{}'>".format(
            self.__class__.__name__, self.id, self.slug, self.name
        )
        return rep.encode('utf-8') if six.PY2 else rep
<commit_msg>Refactor identifiction of changed fields in save() method<commit_after>"""Constants and abstract classes."""
from __future__ import absolute_import, division, print_function, unicode_literals

import copy
import logging
import operator

import six

from .permissions import PermissionsManager

class BaseResource(object):
    """Abstract resource.

    One and only one of the identifiers (slug, id or model_data)
    should be given.

    :param resolwe: Resolwe instance
    :type resolwe: Resolwe object
    :param model_data: Resource model data

    """

    endpoint = None
    query_endpoint = None
    query_method = 'GET'

    WRITABLE_FIELDS = ()
    UPDATE_PROTECTED_FIELDS = ()
    READ_ONLY_FIELDS = ('id',)

    ALL_PERMISSIONS = []  # override this in subclass

    def __init__(self, resolwe, **model_data):
        """Verify that only a single attribute of slug, id or model_data given."""
        self._original_values = {}

        #: unique identifier
        self.id = None  # pylint: disable=invalid-name

        self.api = operator.attrgetter(self.endpoint)(resolwe.api)
        self.resolwe = resolwe

        if model_data:
            self._update_fields(model_data)

        self.logger = logging.getLogger(__name__)

    def fields(self):
        """Resource fields."""
        return self.WRITABLE_FIELDS + self.UPDATE_PROTECTED_FIELDS + self.READ_ONLY_FIELDS

    def _update_fields(self, payload):
        """Update fields of the local resource based on the server values.

        :param dict payload: Resource field values

        """
        self._original_values = copy.deepcopy(payload)
        for field_name in self.fields():
            setattr(self, field_name, payload.get(field_name, None))

    def update(self):
        """Update resource fields from the server."""
        response = self.api(self.id).get()
        self._update_fields(response)

    def _dehydrate_resources(self, obj):
        """Iterate through object and replace all objects with their ids."""
        # Prevent circular imports:
        from .descriptor import DescriptorSchema
        from .process import Process

        if isinstance(obj, DescriptorSchema) or isinstance(obj, Process):
            return obj.slug
        if isinstance(obj, BaseResource):
            return obj.id
        if isinstance(obj, list):
            return [self._dehydrate_resources(element) for element in obj]
        if isinstance(obj, dict):
            return {key: self._dehydrate_resources(value) for key, value in six.iteritems(obj)}
        return obj

    def save(self):
        """Save resource to the server."""
        def field_changed(field_name):
            """Check if local field value is different from the server."""
            return getattr(self, field_name) != self._original_values.get(field_name, None)

        def assert_fields_unchanged(field_names):
            """Assert that fields in ``field_names`` were not changed."""
            changed_fields = [name for name in field_names if field_changed(name)]

            if changed_fields:
                msg = "Not allowed to change read only fields {}".format(', '.join(changed_fields))
                raise ValueError(msg)

        if self.id:  # update resource
            assert_fields_unchanged(self.READ_ONLY_FIELDS + self.UPDATE_PROTECTED_FIELDS)

            payload = {}
            for field_name in self.WRITABLE_FIELDS:
                if field_changed(field_name):
                    payload[field_name] = self._dehydrate_resources(getattr(self, field_name))

            if payload:
                response = self.api(self.id).patch(payload)
                self._update_fields(response)

        else:  # create resource
            assert_fields_unchanged(self.READ_ONLY_FIELDS)

            field_names = self.WRITABLE_FIELDS + self.UPDATE_PROTECTED_FIELDS
            payload = {field_name: self._dehydrate_resources(getattr(self, field_name))
                       for field_name in field_names if getattr(self, field_name) is not None}

            response = self.api.post(payload)
            self._update_fields(response)

    def delete(self, force=False):
        """Delete the resource object from the server."""
        if force is not True:
            user_input = six.moves.input('Do you really want to delete {}?[yN] '.format(self))

            if user_input.strip().lower() != 'y':
                return

        self.api(self.id).delete()

    def __setattr__(self, name, value):
        """Detect changes of read only fields.

        This method detects changes of scalar fields and references. A
        more comprehensive check is called before save.

        """
        if (hasattr(self, '_original_values')
                and name in self._original_values
                and name in self.READ_ONLY_FIELDS
                and value != self._original_values[name]):
            raise ValueError("Can not change read only field {}".format(name))

        super(BaseResource, self).__setattr__(name, value)

    def __eq__(self, obj):
        """Evaluate if objects are the same."""
        if self.__class__ == obj.__class__ and self.id == obj.id:
            return True
        else:
            return False

class BaseResolweResource(BaseResource):
    """Base class for Resolwe resources.

    One and only one of the identifiers (slug, id or model_data)
    should be given.

    :param resolwe: Resolwe instance
    :type resolwe: Resolwe object
    :param model_data: Resource model data

    """

    _permissions = None

    WRITABLE_FIELDS = ('slug', 'name')
    UPDATE_PROTECTED_FIELDS = ('contributor', )
    READ_ONLY_FIELDS = ('id', 'version', 'created', 'modified', 'current_user_permissions')

    def __init__(self, resolwe, **model_data):
        """Initialize attributes."""
        #: a descriptive name of the resource
        self.name = None
        #: the resource version
        self.version = None
        #: human-readable unique identifier
        self.slug = None
        #: user id of the contributor
        self.contributor = None
        #: date of creation
        self.created = None
        #: date of latest modification
        self.modified = None
        #: permissions - (view/download/add/edit/share/owner for user/group/public)
        self.current_user_permissions = None

        BaseResource.__init__(self, resolwe, **model_data)

    @property
    def permissions(self):
        """TODO."""
        if not self.id:
            raise ValueError('Instance must be saved before accessing `permissions` attribute.')
        if not self._permissions:
            self._permissions = PermissionsManager(
                self.ALL_PERMISSIONS,
                self.api(self.id),
                self.resolwe
            )

        return self._permissions

    def update(self):
        """Clear permissions cache and update the object."""
        self.permissions.clear_cache()

        super(BaseResolweResource, self).update()

    def __repr__(self):
        """Format resource name."""
        rep = "{} <id: {}, slug: '{}', name: '{}'>".format(
            self.__class__.__name__, self.id, self.slug, self.name
        )
        return rep.encode('utf-8') if six.PY2 else rep
<|endoftext|>


<commit_before># Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ==============================================================================
from __future__ import print_function

import os
import h5py
import json

import numpy as np
import tensorflow as tf

def log_sum_exp(x_k):
  """Computes log \sum exp in a numerically stable way.
    log ( sum_i exp(x_i) )
    log ( sum_i exp(x_i - m + m) ),       with m = max(x_i)
    log ( sum_i exp(x_i - m)*exp(m) )
    log ( sum_i exp(x_i - m) + m

  Args:
    x_k - k -dimensional list of arguments to log_sum_exp.

  Returns:
    log_sum_exp of the arguments.
  """
  m = tf.reduce_max(x_k)
  x1_k = x_k - m
  u_k = tf.exp(x1_k)
  z = tf.reduce_sum(u_k)
  return tf.log(z) + m

def linear(x, out_size, do_bias=True, alpha=1.0, identity_if_possible=False,
           normalized=False, name=None, collections=None):
  """Linear (affine) transformation, y = x W + b, for a variety of
  configurations.

  Args:
    x: input The tensor to tranformation.
    out_size: The integer size of non-batch output dimension.
    do_bias (optional): Add a learnable bias vector to the operation.
    alpha (optional): A multiplicative scaling for the weight initialization
      of the matrix, in the form \alpha * 1/\sqrt{x.shape[1]}.
    identity_if_possible (optional): just return identity,
      if x.shape[1] == out_size.
    normalized (optional): Option to divide out by the norms of the rows of W.
    name (optional): The name prefix to add to variables.
    collections (optional): List of additional collections. (Placed in
      tf.GraphKeys.GLOBAL_VARIABLES already, so no need for that.)

  Returns:
    In the equation, y = x W + b, returns the tensorflow op that yields y.
  """
  in_size = int(x.get_shape()[1]) # from Dimension(10) -> 10
  stddev = alpha/np.sqrt(float(in_size))
  mat_init = tf.random_normal_initializer(0.0, stddev)
  wname = (name + "/W") if name else "/W"

  if identity_if_possible and in_size == out_size:
    # Sometimes linear layers are nothing more than size adapters.
    return tf.identity(x, name=(wname+'_ident'))

  W,b = init_linear(in_size, out_size, do_bias=do_bias, alpha=alpha,
                    normalized=normalized, name=name, collections=collections)

  if do_bias:
    return tf.matmul(x, W) + b
  else:
    return tf.matmul(x, W)

def init_linear(in_size, out_size, do_bias=True, mat_init_value=None,
                bias_init_value=None, alpha=1.0, identity_if_possible=False,
                normalized=False, name=None, collections=None, trainable=True):
  """Linear (affine) transformation, y = x W + b, for a variety of
  configurations.

  Args:
    in_size: The integer size of the non-batc input dimension. [(x),y]
    out_size: The integer size of non-batch output dimension. [x,(y)]
    do_bias (optional): Add a (learnable) bias vector to the operation,
      if false, b will be an appropriately sized, non-trainable vector
    mat_init_value (optional): numpy constant for matrix initialization, if None
      , do random, with additional parameters.
    alpha (optional): A multiplicative scaling for the weight initialization
      of the matrix, in the form \alpha * 1/\sqrt{x.shape[1]}.
    identity_if_possible (optional): just return identity,
      if x.shape[1] == out_size.
    normalized (optional): Option to divide out by the norms of the rows of W.
    name (optional): The name prefix to add to variables.
    collections (optional): List of additional collections. (Placed in
      tf.GraphKeys.GLOBAL_VARIABLES already, so no need for that.)

  Returns:
    In the equation, y = x W + b, returns the pair (W, b).
  """

  if mat_init_value is not None and mat_init_value.shape != (in_size, out_size):
    raise ValueError(
        'Provided mat_init_value must have shape [%d, %d].'%(in_size, out_size))
  if bias_init_value is not None and bias_init_value.shape != (1,out_size):
    raise ValueError(
        'Provided bias_init_value must have shape [1,%d].'%(out_size,))

  if mat_init_value is None:
    stddev = alpha/np.sqrt(float(in_size))
    mat_init = tf.random_normal_initializer(0.0, stddev)

  wname = (name + "/W") if name else "/W"

  if identity_if_possible and in_size == out_size:
    return (tf.constant(np.eye(in_size).astype(np.float32)),
            tf.zeros(in_size))

  # Note the use of get_variable vs. tf.Variable.  this is because get_variable
  # does not allow the initialization of the variable with a value.
  if normalized:
    w_collections = [tf.GraphKeys.GLOBAL_VARIABLES, "norm-variables"]
    if collections:
      w_collections += collections
    if mat_init_value is not None:
      w = tf.Variable(mat_init_value, name=wname, collections=w_collections,
                      trainable=trainable)
    else:
      w = tf.get_variable(wname, [in_size, out_size], initializer=mat_init,
                          collections=w_collections, trainable=trainable)
    w = tf.nn.l2_normalize(w, dim=0) # x W, so xW_j = \sum_i x_bi W_ij
  else:
    w_collections = [tf.GraphKeys.GLOBAL_VARIABLES]
    if collections:
      w_collections += collections
    if mat_init_value is not None:
      w = tf.Variable(mat_init_value, name=wname, collections=w_collections,
                      trainable=trainable)
    else:
      w = tf.get_variable(wname, [in_size, out_size], initializer=mat_init,
                          collections=w_collections, trainable=trainable)

  b_collections = [tf.GraphKeys.GLOBAL_VARIABLES]
  if collections:
    b_collections += collections
  bname = (name + "/b") if name else "/b"
  if do_bias:
    if bias_init_value is None:
      b = tf.get_variable(bname, [1, out_size],
                          initializer=tf.zeros_initializer(),
                          collections=b_collections, trainable=trainable)
    else:
      b = tf.Variable(bias_init_value, name=bname,
                      collections=b_collections, trainable=trainable)
  else:
      # construct a non-learnable vector of zeros as the bias
      b = tf.get_variable(bname, [1, out_size],
                          initializer=tf.zeros_initializer(),
                          collections=b_collections, trainable=False)

  return (w, b)

def write_data(data_fname, data_dict, use_json=False, compression=None):
  """Write data in HD5F format.

  Args:
    data_fname: The filename of teh file in which to write the data.
    data_dict:  The dictionary of data to write. The keys are strings
      and the values are numpy arrays.
    use_json (optional): human readable format for simple items
    compression (optional): The compression to use for h5py (disabled by
      default because the library borks on scalars, otherwise try 'gzip').
  """

  dir_name = os.path.dirname(data_fname)
  if not os.path.exists(dir_name):
    os.makedirs(dir_name)

  if use_json:
    the_file = open(data_fname,'w')
    json.dump(data_dict, the_file)
    the_file.close()
  else:
    try:
      with h5py.File(data_fname, 'w') as hf:
        for k, v in data_dict.items():
          clean_k = k.replace('/', '_')
          if clean_k is not k:
            print('Warning: saving variable with name: ', k, ' as ', clean_k)
          else:
            print('Saving variable with name: ', clean_k)
          hf.create_dataset(clean_k, data=v, compression=compression)
    except IOError:
      print("Cannot open %s for writing.", data_fname)
      raise

def read_data(data_fname):
  """ Read saved data in HDF5 format.

  Args:
    data_fname: The filename of the file from which to read the data.
  Returns:
    A dictionary whose keys will vary depending on dataset (but should
    always contain the keys 'train_data' and 'valid_data') and whose
    values are numpy arrays.
  """

  try:
    with h5py.File(data_fname, 'r') as hf:
      data_dict = {k: np.array(v) for k, v in hf.items()}
      return data_dict
  except IOError:
    print("Cannot open %s for reading." % data_fname)
    raise

def write_datasets(data_path, data_fname_stem, dataset_dict, compression=None):
  """Write datasets in HD5F format.

  This function assumes the dataset_dict is a mapping ( string ->
  to data_dict ).  It calls write_data for each data dictionary,
  post-fixing the data filename with the key of the dataset.

  Args:
    data_path: The path to the save directory.
    data_fname_stem: The filename stem of the file in which to write the data.
    dataset_dict:  The dictionary of datasets. The keys are strings
      and the values data dictionaries (str -> numpy arrays) associations.
    compression (optional): The compression to use for h5py (disabled by
      default because the library borks on scalars, otherwise try 'gzip').
  """

  full_name_stem = os.path.join(data_path, data_fname_stem)
  for s, data_dict in dataset_dict.items():
    write_data(full_name_stem + "_" + s, data_dict, compression=compression)

def read_datasets(data_path, data_fname_stem):
  """Read dataset sin HD5F format.

  This function assumes the dataset_dict is a mapping ( string ->
  to data_dict ).  It calls write_data for each data dictionary,
  post-fixing the data filename with the key of the dataset.

  Args:
    data_path: The path to the save directory.
    data_fname_stem: The filename stem of the file in which to write the data.
  """

  dataset_dict = {}
  fnames = os.listdir(data_path)

  print ('loading data from ' + data_path + ' with stem ' + data_fname_stem)
  for fname in fnames:
    if fname.startswith(data_fname_stem):
      data_dict = read_data(os.path.join(data_path,fname))
      idx = len(data_fname_stem) + 1
      key = fname[idx:]
      data_dict['data_dim'] = data_dict['train_data'].shape[2]
      data_dict['num_steps'] = data_dict['train_data'].shape[1]
      dataset_dict[key] = data_dict

  if len(dataset_dict) == 0:
    raise ValueError("Failed to load any datasets, are you sure that the "
                     "'--data_dir' and '--data_filename_stem' flag values "
                     "are correct?")

  print (str(len(dataset_dict)) + ' datasets loaded')
  return dataset_dict

# NUMPY utility functions
def list_t_bxn_to_list_b_txn(values_t_bxn):
  """Convert a length T list of BxN numpy tensors of length B list of TxN numpy
  tensors.

  Args:
    values_t_bxn: The length T list of BxN numpy tensors.

  Returns:
    The length B list of TxN numpy tensors.
  """
  T = len(values_t_bxn)
  B, N = values_t_bxn[0].shape
  values_b_txn = []
  for b in range(B):
    values_pb_txn = np.zeros([T,N])
    for t in range(T):
      values_pb_txn[t,:] = values_t_bxn[t][b,:]
    values_b_txn.append(values_pb_txn)

  return values_b_txn

def list_t_bxn_to_tensor_bxtxn(values_t_bxn):
  """Convert a length T list of BxN numpy tensors to single numpy tensor with
  shape BxTxN.

  Args:
    values_t_bxn: The length T list of BxN numpy tensors.

  Returns:
    values_bxtxn: The BxTxN numpy tensor.
  """

  T = len(values_t_bxn)
  B, N = values_t_bxn[0].shape
  values_bxtxn = np.zeros([B,T,N])
  for t in range(T):
    values_bxtxn[:,t,:] = values_t_bxn[t]

  return values_bxtxn

def tensor_bxtxn_to_list_t_bxn(tensor_bxtxn):
  """Convert a numpy tensor with shape BxTxN to a length T list of numpy tensors
  with shape BxT.

  Args:
    tensor_bxtxn: The BxTxN numpy tensor.

  Returns:
    A length T list of numpy tensors with shape BxT.
  """

  values_t_bxn = []
  B, T, N = tensor_bxtxn.shape
  for t in range(T):
    values_t_bxn.append(np.squeeze(tensor_bxtxn[:,t,:]))

  return values_t_bxn

def flatten(list_of_lists):
  """Takes a list of lists and returns a list of the elements.

  Args:
    list_of_lists: List of lists.

  Returns:
    flat_list: Flattened list.
    flat_list_idxs: Flattened list indices.
  """
  flat_list = []
  flat_list_idxs = []
  start_idx = 0
  for item in list_of_lists:
    if isinstance(item, list):
      flat_list += item
      l = len(item)
      idxs = range(start_idx, start_idx+l)
      start_idx = start_idx+l
    else:                   # a value
      flat_list.append(item)
      idxs = [start_idx]
      start_idx += 1
    flat_list_idxs.append(idxs)

  return flat_list, flat_list_idxs
<commit_msg>Restoring old do_bias=False logic, keeping trainable option<commit_after># Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ==============================================================================
from __future__ import print_function

import os
import h5py
import json

import numpy as np
import tensorflow as tf

def log_sum_exp(x_k):
  """Computes log \sum exp in a numerically stable way.
    log ( sum_i exp(x_i) )
    log ( sum_i exp(x_i - m + m) ),       with m = max(x_i)
    log ( sum_i exp(x_i - m)*exp(m) )
    log ( sum_i exp(x_i - m) + m

  Args:
    x_k - k -dimensional list of arguments to log_sum_exp.

  Returns:
    log_sum_exp of the arguments.
  """
  m = tf.reduce_max(x_k)
  x1_k = x_k - m
  u_k = tf.exp(x1_k)
  z = tf.reduce_sum(u_k)
  return tf.log(z) + m

def linear(x, out_size, do_bias=True, alpha=1.0, identity_if_possible=False,
           normalized=False, name=None, collections=None):
  """Linear (affine) transformation, y = x W + b, for a variety of
  configurations.

  Args:
    x: input The tensor to tranformation.
    out_size: The integer size of non-batch output dimension.
    do_bias (optional): Add a learnable bias vector to the operation.
    alpha (optional): A multiplicative scaling for the weight initialization
      of the matrix, in the form \alpha * 1/\sqrt{x.shape[1]}.
    identity_if_possible (optional): just return identity,
      if x.shape[1] == out_size.
    normalized (optional): Option to divide out by the norms of the rows of W.
    name (optional): The name prefix to add to variables.
    collections (optional): List of additional collections. (Placed in
      tf.GraphKeys.GLOBAL_VARIABLES already, so no need for that.)

  Returns:
    In the equation, y = x W + b, returns the tensorflow op that yields y.
  """
  in_size = int(x.get_shape()[1]) # from Dimension(10) -> 10
  stddev = alpha/np.sqrt(float(in_size))
  mat_init = tf.random_normal_initializer(0.0, stddev)
  wname = (name + "/W") if name else "/W"

  if identity_if_possible and in_size == out_size:
    # Sometimes linear layers are nothing more than size adapters.
    return tf.identity(x, name=(wname+'_ident'))

  W,b = init_linear(in_size, out_size, do_bias=do_bias, alpha=alpha,
                    normalized=normalized, name=name, collections=collections)

  if do_bias:
    return tf.matmul(x, W) + b
  else:
    return tf.matmul(x, W)

def init_linear(in_size, out_size, do_bias=True, mat_init_value=None,
                bias_init_value=None, alpha=1.0, identity_if_possible=False,
                normalized=False, name=None, collections=None, trainable=True):
  """Linear (affine) transformation, y = x W + b, for a variety of
  configurations.

  Args:
    in_size: The integer size of the non-batc input dimension. [(x),y]
    out_size: The integer size of non-batch output dimension. [x,(y)]
    do_bias (optional): Add a (learnable) bias vector to the operation,
      if false, b will be an appropriately sized, non-trainable vector
    mat_init_value (optional): numpy constant for matrix initialization, if None
      , do random, with additional parameters.
    alpha (optional): A multiplicative scaling for the weight initialization
      of the matrix, in the form \alpha * 1/\sqrt{x.shape[1]}.
    identity_if_possible (optional): just return identity,
      if x.shape[1] == out_size.
    normalized (optional): Option to divide out by the norms of the rows of W.
    name (optional): The name prefix to add to variables.
    collections (optional): List of additional collections. (Placed in
      tf.GraphKeys.GLOBAL_VARIABLES already, so no need for that.)

  Returns:
    In the equation, y = x W + b, returns the pair (W, b).
  """

  if mat_init_value is not None and mat_init_value.shape != (in_size, out_size):
    raise ValueError(
        'Provided mat_init_value must have shape [%d, %d].'%(in_size, out_size))
  if bias_init_value is not None and bias_init_value.shape != (1,out_size):
    raise ValueError(
        'Provided bias_init_value must have shape [1,%d].'%(out_size,))

  if mat_init_value is None:
    stddev = alpha/np.sqrt(float(in_size))
    mat_init = tf.random_normal_initializer(0.0, stddev)

  wname = (name + "/W") if name else "/W"

  if identity_if_possible and in_size == out_size:
    return (tf.constant(np.eye(in_size).astype(np.float32)),
            tf.zeros(in_size))

  # Note the use of get_variable vs. tf.Variable.  this is because get_variable
  # does not allow the initialization of the variable with a value.
  if normalized:
    w_collections = [tf.GraphKeys.GLOBAL_VARIABLES, "norm-variables"]
    if collections:
      w_collections += collections
    if mat_init_value is not None:
      w = tf.Variable(mat_init_value, name=wname, collections=w_collections,
                      trainable=trainable)
    else:
      w = tf.get_variable(wname, [in_size, out_size], initializer=mat_init,
                          collections=w_collections, trainable=trainable)
    w = tf.nn.l2_normalize(w, dim=0) # x W, so xW_j = \sum_i x_bi W_ij
  else:
    w_collections = [tf.GraphKeys.GLOBAL_VARIABLES]
    if collections:
      w_collections += collections
    if mat_init_value is not None:
      w = tf.Variable(mat_init_value, name=wname, collections=w_collections,
                      trainable=trainable)
    else:
      w = tf.get_variable(wname, [in_size, out_size], initializer=mat_init,
                          collections=w_collections, trainable=trainable)
  b = None
  if do_bias:
    b_collections = [tf.GraphKeys.GLOBAL_VARIABLES]
    if collections:
      b_collections += collections
    bname = (name + "/b") if name else "/b"
    if bias_init_value is None:
      b = tf.get_variable(bname, [1, out_size],
                          initializer=tf.zeros_initializer(),
                          collections=b_collections,
                          trainable=trainable)
    else:
      b = tf.Variable(bias_init_value, name=bname,
                      collections=b_collections,
                      trainable=trainable)

  return (w, b)

def write_data(data_fname, data_dict, use_json=False, compression=None):
  """Write data in HD5F format.

  Args:
    data_fname: The filename of teh file in which to write the data.
    data_dict:  The dictionary of data to write. The keys are strings
      and the values are numpy arrays.
    use_json (optional): human readable format for simple items
    compression (optional): The compression to use for h5py (disabled by
      default because the library borks on scalars, otherwise try 'gzip').
  """

  dir_name = os.path.dirname(data_fname)
  if not os.path.exists(dir_name):
    os.makedirs(dir_name)

  if use_json:
    the_file = open(data_fname,'w')
    json.dump(data_dict, the_file)
    the_file.close()
  else:
    try:
      with h5py.File(data_fname, 'w') as hf:
        for k, v in data_dict.items():
          clean_k = k.replace('/', '_')
          if clean_k is not k:
            print('Warning: saving variable with name: ', k, ' as ', clean_k)
          else:
            print('Saving variable with name: ', clean_k)
          hf.create_dataset(clean_k, data=v, compression=compression)
    except IOError:
      print("Cannot open %s for writing.", data_fname)
      raise

def read_data(data_fname):
  """ Read saved data in HDF5 format.

  Args:
    data_fname: The filename of the file from which to read the data.
  Returns:
    A dictionary whose keys will vary depending on dataset (but should
    always contain the keys 'train_data' and 'valid_data') and whose
    values are numpy arrays.
  """

  try:
    with h5py.File(data_fname, 'r') as hf:
      data_dict = {k: np.array(v) for k, v in hf.items()}
      return data_dict
  except IOError:
    print("Cannot open %s for reading." % data_fname)
    raise

def write_datasets(data_path, data_fname_stem, dataset_dict, compression=None):
  """Write datasets in HD5F format.

  This function assumes the dataset_dict is a mapping ( string ->
  to data_dict ).  It calls write_data for each data dictionary,
  post-fixing the data filename with the key of the dataset.

  Args:
    data_path: The path to the save directory.
    data_fname_stem: The filename stem of the file in which to write the data.
    dataset_dict:  The dictionary of datasets. The keys are strings
      and the values data dictionaries (str -> numpy arrays) associations.
    compression (optional): The compression to use for h5py (disabled by
      default because the library borks on scalars, otherwise try 'gzip').
  """

  full_name_stem = os.path.join(data_path, data_fname_stem)
  for s, data_dict in dataset_dict.items():
    write_data(full_name_stem + "_" + s, data_dict, compression=compression)

def read_datasets(data_path, data_fname_stem):
  """Read dataset sin HD5F format.

  This function assumes the dataset_dict is a mapping ( string ->
  to data_dict ).  It calls write_data for each data dictionary,
  post-fixing the data filename with the key of the dataset.

  Args:
    data_path: The path to the save directory.
    data_fname_stem: The filename stem of the file in which to write the data.
  """

  dataset_dict = {}
  fnames = os.listdir(data_path)

  print ('loading data from ' + data_path + ' with stem ' + data_fname_stem)
  for fname in fnames:
    if fname.startswith(data_fname_stem):
      data_dict = read_data(os.path.join(data_path,fname))
      idx = len(data_fname_stem) + 1
      key = fname[idx:]
      data_dict['data_dim'] = data_dict['train_data'].shape[2]
      data_dict['num_steps'] = data_dict['train_data'].shape[1]
      dataset_dict[key] = data_dict

  if len(dataset_dict) == 0:
    raise ValueError("Failed to load any datasets, are you sure that the "
                     "'--data_dir' and '--data_filename_stem' flag values "
                     "are correct?")

  print (str(len(dataset_dict)) + ' datasets loaded')
  return dataset_dict

# NUMPY utility functions
def list_t_bxn_to_list_b_txn(values_t_bxn):
  """Convert a length T list of BxN numpy tensors of length B list of TxN numpy
  tensors.

  Args:
    values_t_bxn: The length T list of BxN numpy tensors.

  Returns:
    The length B list of TxN numpy tensors.
  """
  T = len(values_t_bxn)
  B, N = values_t_bxn[0].shape
  values_b_txn = []
  for b in range(B):
    values_pb_txn = np.zeros([T,N])
    for t in range(T):
      values_pb_txn[t,:] = values_t_bxn[t][b,:]
    values_b_txn.append(values_pb_txn)

  return values_b_txn

def list_t_bxn_to_tensor_bxtxn(values_t_bxn):
  """Convert a length T list of BxN numpy tensors to single numpy tensor with
  shape BxTxN.

  Args:
    values_t_bxn: The length T list of BxN numpy tensors.

  Returns:
    values_bxtxn: The BxTxN numpy tensor.
  """

  T = len(values_t_bxn)
  B, N = values_t_bxn[0].shape
  values_bxtxn = np.zeros([B,T,N])
  for t in range(T):
    values_bxtxn[:,t,:] = values_t_bxn[t]

  return values_bxtxn

def tensor_bxtxn_to_list_t_bxn(tensor_bxtxn):
  """Convert a numpy tensor with shape BxTxN to a length T list of numpy tensors
  with shape BxT.

  Args:
    tensor_bxtxn: The BxTxN numpy tensor.

  Returns:
    A length T list of numpy tensors with shape BxT.
  """

  values_t_bxn = []
  B, T, N = tensor_bxtxn.shape
  for t in range(T):
    values_t_bxn.append(np.squeeze(tensor_bxtxn[:,t,:]))

  return values_t_bxn

def flatten(list_of_lists):
  """Takes a list of lists and returns a list of the elements.

  Args:
    list_of_lists: List of lists.

  Returns:
    flat_list: Flattened list.
    flat_list_idxs: Flattened list indices.
  """
  flat_list = []
  flat_list_idxs = []
  start_idx = 0
  for item in list_of_lists:
    if isinstance(item, list):
      flat_list += item
      l = len(item)
      idxs = range(start_idx, start_idx+l)
      start_idx = start_idx+l
    else:                   # a value
      flat_list.append(item)
      idxs = [start_idx]
      start_idx += 1
    flat_list_idxs.append(idxs)

  return flat_list, flat_list_idxs
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Django settings used as base for developing wirecloud.

from os import path
from wirecloud.commons.utils.conf import load_default_wirecloud_conf
from wirecloud.commons.utils.urlresolvers import reverse_lazy

DEBUG = True
BASEDIR = path.dirname(path.abspath(__file__))
load_default_wirecloud_conf(locals())

USE_XSENDFILE = False

ADMINS = (
    # ('Your Name', 'your_email@domain.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',      # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': path.join(BASEDIR, 'wirecloud.db'),  # Or path to database file if using sqlite3.
        'USER': '',                                  # Not used with sqlite3.
        'PASSWORD': '',                              # Not used with sqlite3.
        'HOST': '',                                  # Set to empty string for localhost. Not used with sqlite3.
        'PORT': '',                                  # Set to empty string for default. Not used with sqlite3.
    },
}

# This setting has only effect in DJango 1.5+
# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = ['*']

THEME_ACTIVE = "wirecloud.defaulttheme"

# Local time zone for this installation. Choices can be found here:
# http://www.postgresql.org/docs/8.1/static/datetime-keywords.html#DATETIME-TIMEZONE-SET-TABLE
# although not all variations may be possible on all operating systems.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'Europe/Madrid'
DATE_FORMAT = 'd/m/Y'

# Language code for this installation. All choices can be found here:
# http://www.w3.org/TR/REC-html40/struct/dirlang.html#langcodes
LANGUAGE_CODE = 'en'
DEFAULT_LANGUAGE = 'browser'

LANGUAGES = (
    ('es', 'Spanish'),
    ('en', 'English'),
    ('pt', 'Portuguese'),
)

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

STATIC_URL = '/static/'
STATIC_ROOT = path.join(BASEDIR, 'static')
COMPRESS_ROOT = STATIC_ROOT
COMPRESS_OUTPUT_DIR = 'cache'
COMPRESS_JS_FILTERS = (
    'compressor.filters.jsmin.JSMinFilter',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = '15=7f)g=)&spodi3bg8%&4fqt%f3rpg%b$-aer5*#a*(rqm79e'

ROOT_URLCONF = 'urls'

INSTALLED_APPS += (
    'wirecloud.oauth2provider',
    'wirecloud.fiware',
    'django_nose',
)
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'

SESSION_COOKIE_AGE = 5184000  # 2 months

SESSION_EXPIRE_AT_BROWSER_CLOSE = False

# Login/logout URLs
LOGIN_URL = reverse_lazy('login')
LOGOUT_URL = reverse_lazy('wirecloud.root')
LOGIN_REDIRECT_URL = reverse_lazy('wirecloud.root')

#Authentication
AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend',
)

# WGT deployment dirs
CATALOGUE_MEDIA_ROOT = path.join(BASEDIR, 'catalogue', 'media')
GADGETS_DEPLOYMENT_DIR = path.join(BASEDIR, 'deployment', 'widgets')

#SESSION_COOKIE_DOMAIN = '.domain'

# Cache settings
CACHES = {
    'default': {
        'BACKEND': 'wirecloud.platform.cache.backends.locmem.LocMemCache',
        'OPTIONS': {
            'MAX_ENTRIES': 3000,
        },
    }
}

FORCE_SCRIPT_NAME = ""
<commit_msg>Use django.core.urlresolvers.reverse_lazy in the development settings.py file<commit_after># -*- coding: utf-8 -*-
# Django settings used as base for developing wirecloud.

from os import path
from wirecloud.commons.utils.conf import load_default_wirecloud_conf
from django.core.urlresolvers import reverse_lazy

DEBUG = True
BASEDIR = path.dirname(path.abspath(__file__))
load_default_wirecloud_conf(locals())

USE_XSENDFILE = False

ADMINS = (
    # ('Your Name', 'your_email@domain.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',      # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': path.join(BASEDIR, 'wirecloud.db'),  # Or path to database file if using sqlite3.
        'USER': '',                                  # Not used with sqlite3.
        'PASSWORD': '',                              # Not used with sqlite3.
        'HOST': '',                                  # Set to empty string for localhost. Not used with sqlite3.
        'PORT': '',                                  # Set to empty string for default. Not used with sqlite3.
    },
}

# This setting has only effect in DJango 1.5+
# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = ['*']

THEME_ACTIVE = "wirecloud.defaulttheme"

# Local time zone for this installation. Choices can be found here:
# http://www.postgresql.org/docs/8.1/static/datetime-keywords.html#DATETIME-TIMEZONE-SET-TABLE
# although not all variations may be possible on all operating systems.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'Europe/Madrid'
DATE_FORMAT = 'd/m/Y'

# Language code for this installation. All choices can be found here:
# http://www.w3.org/TR/REC-html40/struct/dirlang.html#langcodes
LANGUAGE_CODE = 'en'
DEFAULT_LANGUAGE = 'browser'

LANGUAGES = (
    ('es', 'Spanish'),
    ('en', 'English'),
    ('pt', 'Portuguese'),
)

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

STATIC_URL = '/static/'
STATIC_ROOT = path.join(BASEDIR, 'static')
COMPRESS_ROOT = STATIC_ROOT
COMPRESS_OUTPUT_DIR = 'cache'
COMPRESS_JS_FILTERS = (
    'compressor.filters.jsmin.JSMinFilter',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = '15=7f)g=)&spodi3bg8%&4fqt%f3rpg%b$-aer5*#a*(rqm79e'

ROOT_URLCONF = 'urls'

INSTALLED_APPS += (
    'wirecloud.oauth2provider',
    'wirecloud.fiware',
    'django_nose',
)
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'

SESSION_COOKIE_AGE = 5184000  # 2 months

SESSION_EXPIRE_AT_BROWSER_CLOSE = False

# Login/logout URLs
LOGIN_URL = reverse_lazy('login')
LOGOUT_URL = reverse_lazy('wirecloud.root')
LOGIN_REDIRECT_URL = reverse_lazy('wirecloud.root')

#Authentication
AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend',
)

# WGT deployment dirs
CATALOGUE_MEDIA_ROOT = path.join(BASEDIR, 'catalogue', 'media')
GADGETS_DEPLOYMENT_DIR = path.join(BASEDIR, 'deployment', 'widgets')

#SESSION_COOKIE_DOMAIN = '.domain'

# Cache settings
CACHES = {
    'default': {
        'BACKEND': 'wirecloud.platform.cache.backends.locmem.LocMemCache',
        'OPTIONS': {
            'MAX_ENTRIES': 3000,
        },
    }
}

FORCE_SCRIPT_NAME = ""
<|endoftext|>


<commit_before>#!/usr/bin/python
from vcoptparse import *
import vm_build
import sys
from threading import Thread, Semaphore
import socket, os

class Builder(Thread):
    def __init__(self, name, branch, target, semaphore):
        Thread.__init__(self)
        self.name = name
        self.branch = branch
        self.target = target
        self.semaphore = semaphore
    def run(self):
        self.success = False
        try:
            semaphore.acquire()
            self.target.run(self.branch, self.name)
            self.success = True
        except self.target.RunError, err:
            self.exception = err
        finally:
            semaphore.release()

target_names = ["suse", "redhat5_1", "ubuntu", "debian", "centos5_5", "centos6"]

def help():
    print >>sys.stderr, "Virtual builder:"
    print >>sys.stderr, "     --help      Print this help."
    print >>sys.stderr, "     --target target1 [target2, target3]"
    print >>sys.stderr, "                 Build just one target, options are:"
    print >>sys.stderr, "                 ", target_names
    print >>sys.stderr, "                 defaults to all of them."
    print >>sys.stderr, "     --branch branch_name"
    print >>sys.stderr, "                 Build from a branch mutually exclusive with --tag."
    print >>sys.stderr, "     --tag tag-name"
    print >>sys.stderr, "                 Build from a tag mutually exclusive with --branch."
    print >>sys.stderr, "     --threads number"
    print >>sys.stderr, "                 The number of parallel threads to run."
    print >>sys.stderr, "     --debug"
    print >>sys.stderr, "                 Whether to build the packages with debugging enabled."
    print >>sys.stderr, "     --interact"
    print >>sys.stderr, "                 This starts a target so that you can interact with it."
    print >>sys.stderr, "                 Requires a target."
    print >>sys.stderr, "     --clean-up"
    print >>sys.stderr, "                 Shutdown all running vms"
    print >>sys.stderr, "     --username"
    print >>sys.stderr, "                 Starts the Virtual Machine using VirtualBox from the specified username."
    print >>sys.stderr, "     --hostname"
    print >>sys.stderr, "                 Starts the Virtual Machine using VirtualBox from the specified host machine."

o = OptParser()
o["help"] = BoolFlag("--help")
o["target"] = StringFlag("--target", None)
o["branch"] = StringFlag("--branch", None)
o["tag"] = StringFlag("--tag", None)
o["threads"] = IntFlag("--threads", 3)
o["clean-up"] = BoolFlag("--clean-up")
o["interact"] = BoolFlag("--interact")
o["debug"] = BoolFlag("--debug");
o["username"] = StringFlag("--username", "rethinkdb") # For now, these default values should always be the ones you should use
o["hostname"] = StringFlag("--hostname", "deadshot") # because the UUID values below are hard-coded to correspond with rethinkdb@deadshot

try:
    opts = o.parse(sys.argv)
except OptError:
    print >>sys.stderr, "Argument parsing error"
    help()
    exit(-1)

if opts["help"]:
    help()
    sys.exit(0)

if opts["branch"] and opts["tag"]:
    print >>sys.stderr, "Error cannot use --tag and --branch together."
    help()
    sys.exit(1)

if opts["branch"]:
    rspec = vm_build.Branch(opts["branch"])
elif opts["tag"]:
    rspec = vm_build.Tag(opts["tag"])
else:
    rspec = vm_build.Branch("master")

#Prepare the build flags
flags = "" #this will be given to the makefile
if opts["debug"]:
    flags += " DEBUG=1 UNIT_TESTS=0"
else:
    flags += " DEBUG=0"

suse = vm_build.target('765127b8-2007-43ff-8668-fe4c60176a2b', '192.168.0.173', 'rethinkdb', 'LANG=C make LEGACY_LINUX=1 LEGACY_GCC=1 NO_EVENTFD=1 rpm-suse10 ' + flags, 'rpm', vm_build.rpm_install, vm_build.rpm_uninstall, vm_build.rpm_get_binary, opts["username"], opts["hostname"])
redhat5_1 = vm_build.target('32340f79-cea9-42ca-94d5-2da13d408d02', '192.168.0.159', 'rethinkdb', 'LANG=C make rpm LEGACY_GCC=1 LEGACY_LINUX=1 NO_EVENTFD=1' + flags, 'rpm', vm_build.rpm_install, vm_build.rpm_uninstall, vm_build.rpm_get_binary, opts["username"], opts["hostname"])
ubuntu = vm_build.target('1f4521a0-6e74-4d20-b4b9-9ffd8e231423', '192.168.0.172', 'rethinkdb', 'LANG=C make deb' + flags, 'deb', vm_build.deb_install, vm_build.deb_uninstall, vm_build.deb_get_binary, opts["username"], opts["hostname"])
debian = vm_build.target('cc76e2a5-92c0-4208-be08-5c02429c2c50', '192.168.0.176', 'root', 'LANG=C make deb NO_EVENTFD=1 LEGACY_LINUX=1 ' + flags, 'deb', vm_build.deb_install, vm_build.deb_uninstall, vm_build.deb_get_binary, opts["username"], opts["hostname"])
centos5_5 = vm_build.target('25710682-666f-4449-bd28-68b25abd8bea', '192.168.0.153', 'root', 'LANG=C make rpm LEGACY_GCC=1 LEGACY_LINUX=1 ' + flags, 'rpm', vm_build.rpm_install, vm_build.rpm_uninstall, vm_build.rpm_get_binary, opts["username"], opts["hostname"])
centos6 = vm_build.target('d9058650-a45a-44a5-953f-c2402253a614', '192.168.0.178', 'rethinkdb', 'LANG=C make rpm LEGACY_GCC=1 LEGACY_LINUX=1 ' + flags, 'rpm', vm_build.rpm_install, vm_build.rpm_uninstall, vm_build.rpm_get_binary, opts["username"], opts["hostname"])

targets = {"suse" : suse, "redhat5_1" : redhat5_1, "ubuntu" : ubuntu, "debian" : debian, "centos5_5" : centos5_5, "centos6" : centos6}

if (opts["target"]):
    targets = {opts["target"] : targets[opts["target"]]}

if opts["clean-up"]:
    map(lambda x: x[1].clean_up(), targets.iteritems())
    exit(0)

if opts["interact"]:
    if not opts["target"]:
        print >>sys.stderr, "Error must specify a --target for --interact mode."
        exit(1)
    for name, target in targets.iteritems():
        target.interact(name)
else:
    success = {}
    exception = {}
    semaphore = Semaphore(opts["threads"])

    builders = map(lambda x: Builder(x[0], rspec, x[1], semaphore), targets.iteritems())
    for b in builders:
        x = b.exception
    map(lambda x: x.start(), builders)
    map(lambda x: x.join(), builders)

    for b in builders:
        success[b.name] = b.success
        if not b.success:
            exception[b.name] = b.exception

    print "Build summary:"
    from termcolor import colored
    for name,val in success.iteritems():
        print name, "." * (20 - len(name)), colored("[Pass]", "green") if val else colored("[Fail]", "red")
        if (not val):
            print "Failed on: ", exception[name]
            raise exception[name]

print "Done."
<commit_msg>Undid the last two commits.<commit_after>#!/usr/bin/python
from vcoptparse import *
import vm_build
import sys
from threading import Thread, Semaphore
import socket, os

class Builder(Thread):
    def __init__(self, name, branch, target, semaphore):
        Thread.__init__(self)
        self.name = name
        self.branch = branch
        self.target = target
        self.semaphore = semaphore
    def run(self):
        self.success = False
        try:
            semaphore.acquire()
            self.target.run(self.branch, self.name)
            self.success = True
        except self.target.RunError, err:
            self.exception = err
        finally:
            semaphore.release()

target_names = ["suse", "redhat5_1", "ubuntu", "debian", "centos5_5", "centos6"]

def help():
    print >>sys.stderr, "Virtual builder:"
    print >>sys.stderr, "     --help      Print this help."
    print >>sys.stderr, "     --target target1 [target2, target3]"
    print >>sys.stderr, "                 Build just one target, options are:"
    print >>sys.stderr, "                 ", target_names
    print >>sys.stderr, "                 defaults to all of them."
    print >>sys.stderr, "     --branch branch_name"
    print >>sys.stderr, "                 Build from a branch mutually exclusive with --tag."
    print >>sys.stderr, "     --tag tag-name"
    print >>sys.stderr, "                 Build from a tag mutually exclusive with --branch."
    print >>sys.stderr, "     --threads number"
    print >>sys.stderr, "                 The number of parallel threads to run."
    print >>sys.stderr, "     --debug"
    print >>sys.stderr, "                 Whether to build the packages with debugging enabled."
    print >>sys.stderr, "     --interact"
    print >>sys.stderr, "                 This starts a target so that you can interact with it."
    print >>sys.stderr, "                 Requires a target."
    print >>sys.stderr, "     --clean-up"
    print >>sys.stderr, "                 Shutdown all running vms"
    print >>sys.stderr, "     --username"
    print >>sys.stderr, "                 Starts the Virtual Machine using VirtualBox from the specified username."
    print >>sys.stderr, "     --hostname"
    print >>sys.stderr, "                 Starts the Virtual Machine using VirtualBox from the specified host machine."

o = OptParser()
o["help"] = BoolFlag("--help")
o["target"] = StringFlag("--target", None)
o["branch"] = StringFlag("--branch", None)
o["tag"] = StringFlag("--tag", None)
o["threads"] = IntFlag("--threads", 3)
o["clean-up"] = BoolFlag("--clean-up")
o["interact"] = BoolFlag("--interact")
o["debug"] = BoolFlag("--debug");
o["username"] = StringFlag("--username", "rethinkdb") # For now, these default values should always be the ones you should use
o["hostname"] = StringFlag("--hostname", "deadshot") # because the UUID values below are hard-coded to correspond with rethinkdb@deadshot

try:
    opts = o.parse(sys.argv)
except OptError:
    print >>sys.stderr, "Argument parsing error"
    help()
    exit(-1)

if opts["help"]:
    help()
    sys.exit(0)

if opts["branch"] and opts["tag"]:
    print >>sys.stderr, "Error cannot use --tag and --branch together."
    help()
    sys.exit(1)

if opts["branch"]:
    rspec = vm_build.Branch(opts["branch"])
elif opts["tag"]:
    rspec = vm_build.Tag(opts["tag"])
else:
    rspec = vm_build.Branch("master")

#Prepare the build flags
flags = "" #this will be given to the makefile
if opts["debug"]:
    flags += " DEBUG=1 UNIT_TESTS=0"
else:
    flags += " DEBUG=0"

suse = vm_build.target('765127b8-2007-43ff-8668-fe4c60176a2b', '192.168.0.173', 'rethinkdb', 'LANG=C make LEGACY_LINUX=1 LEGACY_GCC=1 NO_EVENTFD=1 rpm-suse10 ' + flags, 'rpm', vm_build.rpm_install, vm_build.rpm_uninstall, vm_build.rpm_get_binary, opts["username"], opts["hostname"])
redhat5_1 = vm_build.target('32340f79-cea9-42ca-94d5-2da13d408d02', '192.168.0.159', 'rethinkdb', 'LANG=C make rpm LEGACY_GCC=1 LEGACY_LINUX=1 NO_EVENTFD=1' + flags, 'rpm', vm_build.rpm_install, vm_build.rpm_uninstall, vm_build.rpm_get_binary, opts["username"], opts["hostname"])
ubuntu = vm_build.target('1f4521a0-6e74-4d20-b4b9-9ffd8e231423', '192.168.0.172', 'rethinkdb', 'LANG=C make deb' + flags, 'deb', vm_build.deb_install, vm_build.deb_uninstall, vm_build.deb_get_binary, opts["username"], opts["hostname"])
debian = vm_build.target('cc76e2a5-92c0-4208-be08-5c02429c2c50', '192.168.0.176', 'root', 'LANG=C make deb NO_EVENTFD=1 LEGACY_LINUX=1 ' + flags, 'deb', vm_build.deb_install, vm_build.deb_uninstall, vm_build.deb_get_binary, opts["username"], opts["hostname"])
centos5_5 = vm_build.target('25710682-666f-4449-bd28-68b25abd8bea', '192.168.0.153', 'root', 'LANG=C make rpm LEGACY_GCC=1 LEGACY_LINUX=1 ' + flags, 'rpm', vm_build.rpm_install, vm_build.rpm_uninstall, vm_build.rpm_get_binary, opts["username"], opts["hostname"])
centos6 = vm_build.target('d9058650-a45a-44a5-953f-c2402253a614', '192.168.0.178', 'rethinkdb', 'LANG=C make rpm LEGACY_GCC=1 LEGACY_LINUX=1 ' + flags, 'rpm', vm_build.rpm_install, vm_build.rpm_uninstall, vm_build.rpm_get_binary, opts["username"], opts["hostname"])

targets = {"suse" : suse, "redhat5_1" : redhat5_1, "ubuntu" : ubuntu, "debian" : debian, "centos5_5" : centos5_5, "centos6" : centos6}

if (opts["target"]):
    targets = {opts["target"] : targets[opts["target"]]}

if opts["clean-up"]:
    map(lambda x: x[1].clean_up(), targets.iteritems())
    exit(0)

if opts["interact"]:
    if not opts["target"]:
        print >>sys.stderr, "Error must specify a --target for --interact mode."
        exit(1)
    for name, target in targets.iteritems():
        target.interact(name)
else:
    success = {}
    exception = {}
    semaphore = Semaphore(opts["threads"])

    builders = map(lambda x: Builder(x[0], rspec, x[1], semaphore), targets.iteritems())
    map(lambda x: x.start(), builders)
    map(lambda x: x.join(), builders)

    for b in builders:
        success[b.name] = b.success
        if not b.success:
            exception[b.name] = b.exception

    print "Build summary:"
    from termcolor import colored
    for name,val in success.iteritems():
        print name, "." * (20 - len(name)), colored("[Pass]", "green") if val else colored("[Fail]", "red")
        if (not val):
            print "Failed on: ", exception[name]
            raise exception[name]

print "Done."
<|endoftext|>