<commit_before>class Game:
    def __init__(self, logger, players_object, planes_object, plane_positions_object, commands_object, database):
        self.logger = logger
        self.players = players_object
        self.planes = planes_object
        self.planePositions = plane_positions_object
        self.commands = commands_object
        self.database = database
        self.current_mode = "lobby"
        self.current_map = "sta"
        self.best_in_ball = self.database.get_most_goals()
        self.best_in_1dm = self.database.get_most_kills()
        self.best_in_tbd = self.database.get_demolition_expert()
        self.most_goals = self.database.most_goals
        self.most_kills = self.database.most_kills
        self.most_bases_destroyed = self.database.most_bases_destroyed
        self.message_for_best_in_ball = []
        self.message_for_best_in_1dm = []
        self.message_for_best_in_tbd = []
        self.message_for_roundEnd_in_ball = []
        self.message_for_roundEnd_in_1dm = []

    def get_logs_object(self, logs_object):
        self.log = logs_object

    def on_position(self):
        if self.current_map == "asteroids":
            for sublist in self.planePositions.plane_positions:
                x, y = sublist[1]
                if 1780 < x < 1820 and 641 < y < 681:
                    self.commands.Whisper(self.players.nickname_from_id(sublist[0]), "Works so nice!!!")
                    self.log.getPositions = False

    def on_mode_change(self):
        if self.current_mode == "ball":
            self.planes.messagesToSend = ["Press S to use the Ball or any other powerup."]
            if len(self.best_in_ball) != 0:
                for row in self.message_for_best_in_ball:
                    self.planes.messagesToSend.append(row)
            else:
                self.planes.messagesToSend.append('There is no best player of the day in Ball yet.')
                self.planes.messagesToSend.append('Be the first one!')
        elif self.current_mode == "tbd":
            self.planes.messagesToSend = ['Destroy the base with the bomb, press S to use the bomb.']
            if len(self.best_in_tbd) != 0:
                for row in self.message_for_best_in_tbd:
                    self.planes.messagesToSend.append(row)
            else:
                self.planes.messagesToSend.append('There is no best player of the day in TBD yet.')
                self.planes.messagesToSend.append('Be the first one!')
        elif self.current_mode == "1dm":
            self.planes.messagesToSend = ["Kill as many planes as you can and try not to die."]
            if len(self.best_in_1dm) != 0:
                for row in self.message_for_best_in_1dm:
                    self.planes.messagesToSend.append(row)
            else:
                self.planes.messagesToSend.append('There is no best player of the day in 1dm yet.')
                self.planes.messagesToSend.append('Be the first one!')
        elif self.current_mode == "lobby" and self.players.lobby_loaded is False:
            self.planes.messagesToSend = ['This is the lobby, when the are 2 or more players here,',
                                          'use the command "/vote start <gameMode>" in the chat to',
                                          'start a vote for a new match!']
        else:
            self.players.lobby_loaded = False

    def on_map_change(self):
        if self.current_map == "asteroids":
            #self.log.getPositions = True
            #self.planes.messagesToSend.append("This is the most common and loved map in Altitude!")
            return

    def reset_scores(self):
        self.best_in_ball = []
        self.best_in_1dm = []
        self.best_in_tbd = []
        self.most_goals = 0
        self.most_kills = 0
        self.most_bases_destroyed = 0
        self.message_for_best_in_ball = []
        self.message_for_best_in_1dm = []
        self.message_for_best_in_tbd = []
        self.database.reset_values()

    def on_roundEnd(self):
        if self.current_mode == "ball":
            self.commands.Multiple_Messages(self.message_for_roundEnd_in_ball)
        elif self.current_mode == "1dm":
            self.commands.Multiple_Messages(self.message_for_roundEnd_in_1dm)

    def on_goal(self, playerId):
        nickname = self.players.nickname_from_id(playerId)
        self.logger.info("Player's ID who scored goal: {}, vapor: {}".format(playerId, self.players.vapor_from_id(playerId)))
        self.database.add_goal(self.players.vapor_from_id(playerId))
        database_best_in_ball = self.database.get_most_goals()
        database_most_goals = self.database.most_goals
        if len(database_best_in_ball) <= 1:
            if self.best_in_ball != database_best_in_ball:
                self.commands.Multiple_Messages(["New player of the day in Ball:", "{} - {} Goals".format(nickname,
                                                                                database_most_goals)])
                self.message_for_best_in_ball = ['Player of the day in Ball:', '{} - {} Goals'.format(nickname,
                                                                                database_most_goals)]
            if self.best_in_ball == database_best_in_ball:
                if self.most_goals != database_most_goals:
                    self.message_for_roundEnd_in_ball = ["New record from {},", "the player of the day in Ball: {} Goals".format(nickname,
                                                                                database_most_goals)]
                    self.message_for_best_in_ball = ['Player of the day in Ball:', '{} - {} Goals'.format(nickname,
                                                                                database_most_goals)]
        elif len(database_best_in_ball) > 1:
            players = len(database_best_in_ball)
            self.message_for_best_in_ball = []
            self.message_for_roundEnd_in_ball = []
            if self.best_in_ball != database_best_in_ball:
                self.message_for_roundEnd_in_ball.append("There are now {} best players of the day in Ball with {} Goals:".format(players, database_most_goals))
                self.message_for_best_in_ball.append("Players of the day in Ball with {} Goals:".format(database_most_goals))
                for number in range(players):
                    self.message_for_roundEnd_in_ball.append("{}) {}".format(number+1, database_best_in_ball[number]))
                    self.message_for_best_in_ball.append("{}) {}".format(number+1, database_best_in_ball[number]))
        self.best_in_ball = database_best_in_ball
        self.most_goals = database_most_goals

    def on_base_destroy(self, playerId):
        nickname = self.players.nickname_from_id(playerId)
        self.database.add_base(self.players.vapor_from_id(playerId))
        database_best_in_tbd = self.database.get_demolition_expert()
        database_most_bases_destroyed = self.database.most_bases_destroyed
        if len(database_best_in_tbd) <= 1:
            if self.best_in_tbd != database_best_in_tbd:
                self.commands.Multiple_Messages(["New player of the day in TBD:", "{} - {} Bases Destroyed".format(nickname,
                                                                                database_most_bases_destroyed)])
                self.message_for_best_in_tbd = ['Player of the day in TBD:', '{} - {} Bases Destroyed'.format(nickname,
                                                                                database_most_bases_destroyed)]
            if self.best_in_tbd == database_best_in_tbd:
                if self.most_bases_destroyed != database_most_bases_destroyed:
                    self.commands.Multiple_Messages(["New record from {},", "the player of the day in TBD: {} Bases Destroyed".format(nickname,
                                                                                database_most_bases_destroyed)])
                    self.message_for_best_in_tbd = ['Player of the day in TBD:', '{} - {} Bases Destroyed'.format(nickname,
                                                                                database_most_bases_destroyed)]
        elif len(database_best_in_tbd) > 1:
            players = len(database_best_in_tbd)
            self.message_for_best_in_tbd = []
            if self.best_in_tbd != database_best_in_tbd:
                self.commands.Message("There are now {} players of the day in TBD with {} Bases Destroyed:".format(players,
                                                                                database_most_bases_destroyed))
                self.message_for_best_in_tbd.append(
                    "Players of the day in TBD with {} Bases Destroyed:".format(database_most_bases_destroyed))
                for number in range(players):
                    self.commands.Message("{}) {}".format(number + 1, database_best_in_tbd[number]))
                    self.message_for_best_in_tbd.append("{}) {}".format(number + 1, database_best_in_tbd[number]))
        self.best_in_tbd = database_best_in_tbd
        self.most_bases_destroyed = database_most_bases_destroyed

    def on_kill(self, playerId):
        if self.current_mode == "1dm":
            nickname = self.players.nickname_from_id(playerId)
            self.database.add_kill(self.players.vapor_from_id(playerId))
            database_best_in_1dm = self.database.get_most_kills()
            database_most_kills = self.database.most_kills
            if len(database_best_in_1dm) <= 1:
                if self.best_in_1dm != database_best_in_1dm:
                    self.commands.Multiple_Messages(["New player of the day in 1dm:", "{} - {} Kills".format(nickname,
                                                                                database_best_in_1dm)])
                    self.message_for_best_in_1dm = ['Player of the day in 1dm:', '{} - {} Kills'.format(nickname,
                                                                                database_most_kills)]
                if self.best_in_1dm == database_best_in_1dm:
                    if self.most_kills != database_most_kills:
                        self.message_for_roundEnd_in_1dm = ["New record from {},", "the player of the day in 1dm: {} Kills".format(nickname,
                                                                                database_most_kills)]
                        self.message_for_best_in_1dm = ['Player of the day in 1dm:', '{} - {} Kills'.format(nickname,
                                                                                database_most_kills)]
            elif len(database_best_in_1dm) > 1:
                players = len(database_best_in_1dm)
                self.message_for_best_in_1dm = []
                self.message_for_roundEnd_in_1dm = []
                if self.best_in_1dm != database_best_in_1dm:
                    self.message_for_roundEnd_in_1dm.append("There are now {} players of the day in 1dm with {} Kills:".format(players, database_most_kills))
                    self.message_for_best_in_1dm.append("Players of the day in 1dm with {} Kills:".format(database_most_kills))
                    for number in range(players):
                        self.message_for_roundEnd_in_1dm.append("{}) {}".format(number+1, database_best_in_1dm[number]))
                        self.message_for_best_in_1dm.append("{}) {}".format(number+1, database_best_in_1dm[number]))
            self.best_in_1dm = database_best_in_1dm
            self.most_kills = database_most_kills

    def check_current_mode_and_map(self, full_map):
        mode, mapName = full_map.split("_")
        if mode != self.current_mode:
            self.current_mode = mode
            self.on_mode_change()
        if mapName != self.current_map:
            self.current_map = mapName
            self.on_map_change()<commit_msg>Fixed bug for not showing roundEnd message properly<commit_after>class Game:
    def __init__(self, logger, players_object, planes_object, plane_positions_object, commands_object, database):
        self.logger = logger
        self.players = players_object
        self.planes = planes_object
        self.planePositions = plane_positions_object
        self.commands = commands_object
        self.database = database
        self.current_mode = "lobby"
        self.current_map = "sta"
        self.best_in_ball = self.database.get_most_goals()
        self.best_in_1dm = self.database.get_most_kills()
        self.best_in_tbd = self.database.get_demolition_expert()
        self.most_goals = self.database.most_goals
        self.most_kills = self.database.most_kills
        self.most_bases_destroyed = self.database.most_bases_destroyed
        self.message_for_best_in_ball = []
        self.message_for_best_in_1dm = []
        self.message_for_best_in_tbd = []
        self.message_for_roundEnd_in_ball = []
        self.message_for_roundEnd_in_1dm = []

    def get_logs_object(self, logs_object):
        self.log = logs_object

    def on_position(self):
        if self.current_map == "asteroids":
            for sublist in self.planePositions.plane_positions:
                x, y = sublist[1]
                if 1780 < x < 1820 and 641 < y < 681:
                    self.commands.Whisper(self.players.nickname_from_id(sublist[0]), "Works so nice!!!")
                    self.log.getPositions = False

    def on_mode_change(self):
        if self.current_mode == "ball":
            self.planes.messagesToSend = ["Press S to use the Ball or any other powerup."]
            if len(self.best_in_ball) != 0:
                for row in self.message_for_best_in_ball:
                    self.planes.messagesToSend.append(row)
            else:
                self.planes.messagesToSend.append('There is no best player of the day in Ball yet.')
                self.planes.messagesToSend.append('Be the first one!')
        elif self.current_mode == "tbd":
            self.planes.messagesToSend = ['Destroy the base with the bomb, press S to use the bomb.']
            if len(self.best_in_tbd) != 0:
                for row in self.message_for_best_in_tbd:
                    self.planes.messagesToSend.append(row)
            else:
                self.planes.messagesToSend.append('There is no best player of the day in TBD yet.')
                self.planes.messagesToSend.append('Be the first one!')
        elif self.current_mode == "1dm":
            self.planes.messagesToSend = ["Kill as many planes as you can and try not to die."]
            if len(self.best_in_1dm) != 0:
                for row in self.message_for_best_in_1dm:
                    self.planes.messagesToSend.append(row)
            else:
                self.planes.messagesToSend.append('There is no best player of the day in 1dm yet.')
                self.planes.messagesToSend.append('Be the first one!')
        elif self.current_mode == "lobby" and self.players.lobby_loaded is False:
            self.planes.messagesToSend = ['This is the lobby, when the are 2 or more players here,',
                                          'use the command "/vote start <gameMode>" in the chat to',
                                          'start a vote for a new match!']
        else:
            self.players.lobby_loaded = False

    def on_map_change(self):
        if self.current_map == "asteroids":
            #self.log.getPositions = True
            #self.planes.messagesToSend.append("This is the most common and loved map in Altitude!")
            return

    def reset_scores(self):
        self.best_in_ball = []
        self.best_in_1dm = []
        self.best_in_tbd = []
        self.most_goals = 0
        self.most_kills = 0
        self.most_bases_destroyed = 0
        self.message_for_best_in_ball = []
        self.message_for_best_in_1dm = []
        self.message_for_best_in_tbd = []
        self.database.reset_values()

    def on_roundEnd(self):
        if self.current_mode == "ball":
            self.commands.Multiple_Messages(self.message_for_roundEnd_in_ball)
        elif self.current_mode == "1dm":
            self.commands.Multiple_Messages(self.message_for_roundEnd_in_1dm)

    def on_goal(self, playerId):
        nickname = self.players.nickname_from_id(playerId)
        self.logger.info("Player's ID who scored goal: {}, vapor: {}".format(playerId, self.players.vapor_from_id(playerId)))
        self.database.add_goal(self.players.vapor_from_id(playerId))
        database_best_in_ball = self.database.get_most_goals()
        database_most_goals = self.database.most_goals
        if len(database_best_in_ball) <= 1:
            if self.best_in_ball != database_best_in_ball:
                self.commands.Multiple_Messages(["New player of the day in Ball:", "{} - {} Goals".format(nickname,
                                                                                database_most_goals)])
                self.message_for_best_in_ball = ['Player of the day in Ball:', '{} - {} Goals'.format(nickname,
                                                                                database_most_goals)]
            if self.best_in_ball == database_best_in_ball:
                if self.most_goals != database_most_goals:
                    self.message_for_roundEnd_in_ball = ["New record from {},".format(nickname), "the player of the day in Ball: {} Goals".format(
                                                                                database_most_goals)]
                    self.message_for_best_in_ball = ['Player of the day in Ball:', '{} - {} Goals'.format(nickname,
                                                                                database_most_goals)]
        elif len(database_best_in_ball) > 1:
            players = len(database_best_in_ball)
            self.message_for_best_in_ball = []
            self.message_for_roundEnd_in_ball = []
            if self.best_in_ball != database_best_in_ball:
                self.message_for_roundEnd_in_ball.append("There are now {} best players of the day in Ball with {} Goals:".format(players, database_most_goals))
                self.message_for_best_in_ball.append("Players of the day in Ball with {} Goals:".format(database_most_goals))
                for number in range(players):
                    self.message_for_roundEnd_in_ball.append("{}) {}".format(number+1, database_best_in_ball[number]))
                    self.message_for_best_in_ball.append("{}) {}".format(number+1, database_best_in_ball[number]))
        self.best_in_ball = database_best_in_ball
        self.most_goals = database_most_goals

    def on_base_destroy(self, playerId):
        nickname = self.players.nickname_from_id(playerId)
        self.database.add_base(self.players.vapor_from_id(playerId))
        database_best_in_tbd = self.database.get_demolition_expert()
        database_most_bases_destroyed = self.database.most_bases_destroyed
        if len(database_best_in_tbd) <= 1:
            if self.best_in_tbd != database_best_in_tbd:
                self.commands.Multiple_Messages(["New player of the day in TBD:", "{} - {} Bases Destroyed".format(nickname,
                                                                                database_most_bases_destroyed)])
                self.message_for_best_in_tbd = ['Player of the day in TBD:', '{} - {} Bases Destroyed'.format(nickname,
                                                                                database_most_bases_destroyed)]
            if self.best_in_tbd == database_best_in_tbd:
                if self.most_bases_destroyed != database_most_bases_destroyed:
                    self.commands.Multiple_Messages(["New record from {},".format(nickname), "the player of the day in TBD: {} Bases Destroyed".format(
                                                                                database_most_bases_destroyed)])
                    self.message_for_best_in_tbd = ['Player of the day in TBD:', '{} - {} Bases Destroyed'.format(nickname,
                                                                                database_most_bases_destroyed)]
        elif len(database_best_in_tbd) > 1:
            players = len(database_best_in_tbd)
            self.message_for_best_in_tbd = []
            if self.best_in_tbd != database_best_in_tbd:
                self.commands.Message("There are now {} players of the day in TBD with {} Bases Destroyed:".format(players,
                                                                                database_most_bases_destroyed))
                self.message_for_best_in_tbd.append(
                    "Players of the day in TBD with {} Bases Destroyed:".format(database_most_bases_destroyed))
                for number in range(players):
                    self.commands.Message("{}) {}".format(number + 1, database_best_in_tbd[number]))
                    self.message_for_best_in_tbd.append("{}) {}".format(number + 1, database_best_in_tbd[number]))
        self.best_in_tbd = database_best_in_tbd
        self.most_bases_destroyed = database_most_bases_destroyed

    def on_kill(self, playerId):
        if self.current_mode == "1dm":
            nickname = self.players.nickname_from_id(playerId)
            self.database.add_kill(self.players.vapor_from_id(playerId))
            database_best_in_1dm = self.database.get_most_kills()
            database_most_kills = self.database.most_kills
            if len(database_best_in_1dm) <= 1:
                if self.best_in_1dm != database_best_in_1dm:
                    self.commands.Multiple_Messages(["New player of the day in 1dm:", "{} - {} Kills".format(nickname,
                                                                                database_best_in_1dm)])
                    self.message_for_best_in_1dm = ['Player of the day in 1dm:', '{} - {} Kills'.format(nickname,
                                                                                database_most_kills)]
                if self.best_in_1dm == database_best_in_1dm:
                    if self.most_kills != database_most_kills:
                        self.message_for_roundEnd_in_1dm = ["New record from {},".format(nickname), "the player of the day in 1dm: {} Kills".format(
                                                                                database_most_kills)]
                        self.message_for_best_in_1dm = ['Player of the day in 1dm:', '{} - {} Kills'.format(nickname,
                                                                                database_most_kills)]
            elif len(database_best_in_1dm) > 1:
                players = len(database_best_in_1dm)
                self.message_for_best_in_1dm = []
                self.message_for_roundEnd_in_1dm = []
                if self.best_in_1dm != database_best_in_1dm:
                    self.message_for_roundEnd_in_1dm.append("There are now {} players of the day in 1dm with {} Kills:".format(players, database_most_kills))
                    self.message_for_best_in_1dm.append("Players of the day in 1dm with {} Kills:".format(database_most_kills))
                    for number in range(players):
                        self.message_for_roundEnd_in_1dm.append("{}) {}".format(number+1, database_best_in_1dm[number]))
                        self.message_for_best_in_1dm.append("{}) {}".format(number+1, database_best_in_1dm[number]))
            self.best_in_1dm = database_best_in_1dm
            self.most_kills = database_most_kills

    def check_current_mode_and_map(self, full_map):
        mode, mapName = full_map.split("_")
        if mode != self.current_mode:
            self.current_mode = mode
            self.on_mode_change()
        if mapName != self.current_map:
            self.current_map = mapName
            self.on_map_change()<|endoftext|>


<commit_before>from fabric.api import abort, env, local, put, roles, run, sudo
import re

env.roledefs = {'slave': ['1.cdn.libravatar.org', '2.cdn.libravatar.org', '3.cdn.libravatar.org'],
                'master': ['0.cdn.libravatar.org']}

COMMON_PACKAGES = ['libravatar-cdn', 'libravatar-common', 'libravatar-cdn-common']
SLAVE_PACKAGES = ['libravatar-seccdn', 'libravatar-slave']
MASTER_PACKAGES = ['libravatar', 'libravatar-www', 'libravatar-master']

# Extract current version number from Debian changelog
with open('debian/changelog', 'r') as changelog:
    PACKAGE_VERSION = re.search('\(([0-9]+\.[0-9])\)', changelog.readline()).group(1)

def commit_changelog():
    if local("bzr status --short", capture=True) != 'M  debian/changelog':
        abort("You must first update the Debian changelog.")

    local('bzr commit -m "Bump changelog for deployment"', capture=False)

def prepare():
    local('make clean', capture=False)
    commit_changelog()
    local('make package', capture=False)

def copy_and_install_packages(package_names):
    all_debs = ''
    for package_name in package_names:
        deb = '%s_%s_all.deb' % (package_name, PACKAGE_VERSION)
        all_debs += ' /home/%s/debs/%s' % (env.user, deb)
        put('../%s' % deb, 'debs/')

    sudo('/usr/bin/dpkg -i%s' % all_debs, shell=False)

def restart_apache():
    # Restart Apache to make mod_wsgi use the new files
    sudo('/usr/sbin/apache2ctl configtest', shell=False)
    sudo('/usr/sbin/apache2ctl graceful', shell=False)

def commit_etc_changes():
    # TODO: deal with etckeeper (check that the git branch is clean, then commit automatically)
    pass

@roles('slave')
def deploy_slave():
    run('mkdir -p debs')  # ensure the target directory exists
    copy_and_install_packages(COMMON_PACKAGES)
    copy_and_install_packages(SLAVE_PACKAGES)
    restart_apache()
    commit_etc_changes()

@roles('master')
def deploy_master():
    run('mkdir -p debs')  # ensure the target directory exists
    copy_and_install_packages(COMMON_PACKAGES)
    copy_and_install_packages(MASTER_PACKAGES)
    restart_apache()
    commit_etc_changes()

def deploy():
    deploy_slave()
    deploy_master()

def stage():
    pass  # TODO: deploy all of the packages to a staging server
<commit_msg>Deploy to the slaves in parallel (with fallback for older fabric)<commit_after>from fabric.api import abort, env, local, put, roles, run, sudo
import re

try:
    # This will fail on Squeeze's version of fabric
    from fabric.api import parallel
except ImportError:
    # Define a stub decorator that doesn't do anything
    def parallel(function):
        return function

env.roledefs = {'slave': ['1.cdn.libravatar.org', '2.cdn.libravatar.org', '3.cdn.libravatar.org'],
                'master': ['0.cdn.libravatar.org']}

COMMON_PACKAGES = ['libravatar-cdn', 'libravatar-common', 'libravatar-cdn-common']
SLAVE_PACKAGES = ['libravatar-seccdn', 'libravatar-slave']
MASTER_PACKAGES = ['libravatar', 'libravatar-www', 'libravatar-master']

# Extract current version number from Debian changelog
with open('debian/changelog', 'r') as changelog:
    PACKAGE_VERSION = re.search('\(([0-9]+\.[0-9])\)', changelog.readline()).group(1)

def commit_changelog():
    if local("bzr status --short", capture=True) != 'M  debian/changelog':
        abort("You must first update the Debian changelog.")

    local('bzr commit -m "Bump changelog for deployment"', capture=False)

def prepare():
    local('make clean', capture=False)
    commit_changelog()
    local('make package', capture=False)

def copy_and_install_packages(package_names):
    all_debs = ''
    for package_name in package_names:
        deb = '%s_%s_all.deb' % (package_name, PACKAGE_VERSION)
        all_debs += ' /home/%s/debs/%s' % (env.user, deb)
        put('../%s' % deb, 'debs/')

    sudo('/usr/bin/dpkg -i%s' % all_debs, shell=False)

def restart_apache():
    # Restart Apache to make mod_wsgi use the new files
    sudo('/usr/sbin/apache2ctl configtest', shell=False)
    sudo('/usr/sbin/apache2ctl graceful', shell=False)

def commit_etc_changes():
    # TODO: deal with etckeeper (check that the git branch is clean, then commit automatically)
    pass

@parallel
@roles('slave')
def deploy_slave():
    run('mkdir -p debs')  # ensure the target directory exists
    copy_and_install_packages(COMMON_PACKAGES)
    copy_and_install_packages(SLAVE_PACKAGES)
    restart_apache()
    commit_etc_changes()

@roles('master')
def deploy_master():
    run('mkdir -p debs')  # ensure the target directory exists
    copy_and_install_packages(COMMON_PACKAGES)
    copy_and_install_packages(MASTER_PACKAGES)
    restart_apache()
    commit_etc_changes()

def deploy():
    deploy_slave()
    deploy_master()

def stage():
    pass  # TODO: deploy all of the packages to a staging server
<|endoftext|>


<commit_before>"""empty message

Revision ID: 0014_add_template_version
Revises: 0013_add_loadtest_client
Create Date: 2016-05-11 16:00:51.478012

"""

# revision identifiers, used by Alembic.
revision = '0014_add_template_version'
down_revision = '0013_add_loadtest_client'

from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

def upgrade():
    op.add_column('jobs', sa.Column('template_version', sa.Integer(), nullable=True))
    op.get_bind()
    op.execute('update jobs set template_version = (select version from templates where id = template_id)')
    op.add_column('notifications', sa.Column('template_version', sa.Integer(), nullable=True))
    op.execute('update notifications set template_version = (select version from templates where id = template_id)')
    op.alter_column('jobs', 'template_version', nullable=False)
    op.alter_column('notifications', 'template_version', nullable=False)

def downgrade():
    op.drop_column('notifications', 'template_version')
    op.drop_column('jobs', 'template_version')
<commit_msg>Add a update query to fix templates_history where the created_by_id is missing.<commit_after>"""empty message

Revision ID: 0014_add_template_version
Revises: 0013_add_loadtest_client
Create Date: 2016-05-11 16:00:51.478012

"""

# revision identifiers, used by Alembic.
revision = '0014_add_template_version'
down_revision = '0013_add_loadtest_client'

from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

def upgrade():
    op.add_column('jobs', sa.Column('template_version', sa.Integer(), nullable=True))
    op.get_bind()
    op.execute('update jobs set template_version = (select version from templates where id = template_id)')
    op.add_column('notifications', sa.Column('template_version', sa.Integer(), nullable=True))
    op.execute('update notifications set template_version = (select version from templates where id = template_id)')
    op.alter_column('jobs', 'template_version', nullable=False)
    op.alter_column('notifications', 'template_version', nullable=False)

    # fix template_history where created_by_id is not set.
    query = "update templates_history set created_by_id = " \
            "         (select created_by_id from templates " \
            "           where templates.id = templates_history.id " \
            "           and templates.version = templates_history.version) " \
            "where templates_history.created_by_id is null"
    op.execute(query)

def downgrade():
    op.drop_column('notifications', 'template_version')
    op.drop_column('jobs', 'template_version')
<|endoftext|>


<commit_before># fabricfile to deploy build
#
# depends on installation of fabric - pip install fabric
#
# example invocation
# $ fab -H jenkins@uf04.seedscientific.com deploy
# $ fab -H ubuntu@52.0.138.67 deploy

from fabric.api import local, run, cd, put
from fabvenv import virtualenv, make_virtualenv

## global variables
##
local_venv_path = '/tmp/venv'
# remote_venv_path = '/tmp/venv'

# /var/www/clients.seedscientific.com/uf/UF04
remote_work_path = '~/deploy/polio-work'
remote_backend_path = '/var/www/polio/'
remote_frontend_path = '/var/www/polio/static/'

# test build
#
# test-machine dependencies - python, pip, postgres
#
def test():
    local("echo TODO: do tests here")

# deploy build
#
# build-machine dependencies - node, gulp, bower, sass, compass, ruby, virtualenv, fabric-virtualenv
def deploy():
    ###
    ### on build machine...
    ###

    # set up dependencies
    print ("TODO: confirm build machine has dependencies. i.e. node, gulp.")
    # e.g.
    # sudo gem install sass
    # sudo gem install compass

    # make virtual env
    make_virtualenv(local_venv_path)

    # enter virtual environment
    with virtualenv(local_venv_path):
        local ("pip install -r requirements.txt")

        # make dist
        local("./node_modules/.bin/bower install")
        local("./node_modules/.bin/gulp dist")

    ###
    ### on target machine...
    ###

    # make folder if it doesn't exist
    run ("mkdir -p %s" % remote_work_path)

    # push to remote server
    put ('dist/uf04-frontend.zip', remote_work_path)
    put ('dist/uf04-backend.zip', remote_work_path)

    # unzip stuff
    with cd(remote_work_path):
        run("unzip -o uf04-frontend.zip -d %s" % remote_frontend_path) # -o is overwrite
        run("unzip -o uf04-backend.zip -d %s" % remote_backend_path)

    # in front-end path
    with cd(remote_backend_path):
        run("chgrp -R www-data .")

    # in server path -
    with cd(remote_backend_path):
        run("pip install -r requirements.txt")

        # echo "== SYNCDB / MIGRATE =="
        run("python manage.py syncdb --noinput --settings=polio.prod_settings")
        run("python manage.py migrate --noinput --settings=polio.prod_settings")

        # echo "== BUILDING DATABASE =="
        run("bash bin/build_db.sh")

    # bounce apache??
    # customize any other configuration?
    #
    # echo "== BUILDING DOCUMENTATION ==" # maybe...
    # make clean -C docs
    # make html -C docs
    #
    # echo "== RUNNING TESTS =="
    # python manage.py test datapoints.tests.test_cache --settings=polio.settings_test

# def prepare_deploy():
    # local("pip install -r requirements.txt")

    # from shell script
    # git pull origin development
    # pip install -r requirements.txt
    # python manage.py syncdb --settings=polio.prod_settings
    # python manage.py migrate --settings=polio.prod_settings
    # bash bin/build_db.sh
<commit_msg>add comment w/path to test server<commit_after># fabricfile to deploy build
#
# depends on installation of fabric - pip install fabric
#
# example invocation
# $ fab -H jenkins@uf04.seedscientific.com deploy
# $ fab -H ubuntu@52.0.138.67 deploy
# $ fab -H ubuntu@uf04.seedscientific.com deploy

from fabric.api import local, run, cd, put
from fabvenv import virtualenv, make_virtualenv

## global variables
##
local_venv_path = '/tmp/venv'
# remote_venv_path = '/tmp/venv'

# /var/www/clients.seedscientific.com/uf/UF04
remote_work_path = '~/deploy/polio-work'
remote_backend_path = '/var/www/polio/'
remote_frontend_path = '/var/www/polio/static/'

# test build
#
# test-machine dependencies - python, pip, postgres
#
def test():
    local("echo TODO: do tests here")

# deploy build
#
# build-machine dependencies - node, gulp, bower, sass, compass, ruby, virtualenv, fabric-virtualenv
def deploy():
    ###
    ### on build machine...
    ###

    # set up dependencies
    print ("TODO: confirm build machine has dependencies. i.e. node, gulp.")
    # e.g.
    # sudo gem install sass
    # sudo gem install compass

    # make virtual env
    make_virtualenv(local_venv_path)

    # enter virtual environment
    with virtualenv(local_venv_path):
        local ("pip install -r requirements.txt")

        # make dist
        local("./node_modules/.bin/bower install")
        local("./node_modules/.bin/gulp dist")

    ###
    ### on target machine...
    ###

    # make folder if it doesn't exist
    run ("mkdir -p %s" % remote_work_path)

    # push to remote server
    put ('dist/uf04-frontend.zip', remote_work_path)
    put ('dist/uf04-backend.zip', remote_work_path)

    # unzip stuff
    with cd(remote_work_path):
        run("unzip -o uf04-frontend.zip -d %s" % remote_frontend_path) # -o is overwrite
        run("unzip -o uf04-backend.zip -d %s" % remote_backend_path)

    # in front-end path
    with cd(remote_backend_path):
        run("chgrp -R www-data .")

    # in server path -
    with cd(remote_backend_path):
        run("pip install -r requirements.txt")

        # echo "== SYNCDB / MIGRATE =="
        run("python manage.py syncdb --noinput --settings=polio.prod_settings")
        run("python manage.py migrate --noinput --settings=polio.prod_settings")

        # echo "== BUILDING DATABASE =="
        run("bash bin/build_db.sh")

    # bounce apache??
    # customize any other configuration?
    #
    # echo "== BUILDING DOCUMENTATION ==" # maybe...
    # make clean -C docs
    # make html -C docs
    #
    # echo "== RUNNING TESTS =="
    # python manage.py test datapoints.tests.test_cache --settings=polio.settings_test

# def prepare_deploy():
    # local("pip install -r requirements.txt")

    # from shell script
    # git pull origin development
    # pip install -r requirements.txt
    # python manage.py syncdb --settings=polio.prod_settings
    # python manage.py migrate --settings=polio.prod_settings
    # bash bin/build_db.sh
<|endoftext|>


<commit_before>from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals
from __future__ import absolute_import
from builtins import *  # NOQA
from future import standard_library
standard_library.install_aliases()  # NOQA
import argparse
import functools
import os

import chainer
from chainer import functions as F
from chainer import links as L
from chainer import optimizers
import gym
import gym.spaces
import numpy as np

import chainerrl
from chainerrl.action_value import DiscreteActionValue
from chainerrl import experiments
from chainerrl import explorers
from chainerrl import misc
from chainerrl import replay_buffer

class CastAction(gym.ActionWrapper):
    """Cast actions to a given type."""

    def __init__(self, env, type_):
        super().__init__(env)
        self.type_ = type_

    def _action(self, action):
        return self.type_(action)

class TransposeObservation(gym.ObservationWrapper):
    """Transpose observations."""

    def __init__(self, env, axes):
        super().__init__(env)
        self._axes = axes

    def _observation(self, observation):
        return observation.transpose(*self._axes)

class ObserveElapsedSteps(gym.Wrapper):
    """Observe the number of elapsed steps in an episode.

    A new observation will be a tuple of an original observation and an integer
    that is equal to the elapsed steps in an episode.
    """

    def __init__(self, env, max_steps):
        super().__init__(env)
        self._max_steps = max_steps
        self._elapsed_steps = 0
        self.observation_space = gym.spaces.Tuple((
            env.observation_space,
            gym.spaces.Discrete(self._max_steps + 1),
        ))

    def reset(self):
        self._elapsed_steps = 0
        return self.env.reset(), self._elapsed_steps

    def _step(self, action):
        observation, reward, done, info = self.env.step(action)
        self._elapsed_steps += 1
        assert self._elapsed_steps <= self._max_steps
        return (observation, self._elapsed_steps), reward, done, info

class RecordMovie(gym.Wrapper):
    """Record MP4 videos using pybullet's logging API."""

    def __init__(self, env, dirname):
        super().__init__(env)
        self._episode_idx = -1
        self._dirname = dirname

    def reset(self):
        obs = self.env.reset()
        self._episode_idx += 1
        import pybullet
        pybullet.startStateLogging(
            pybullet.STATE_LOGGING_VIDEO_MP4,
            os.path.join(self._dirname, '{}.mp4'.format(self._episode_idx)))
        return obs

class GraspingQFunction(chainer.Chain):
    """Q-function model for the grasping env.

    This model takes an 84x84 2D image and an integer that indicates the
    number of elapsed steps in an episode as input and outputs action values.
    """

    def __init__(self, n_actions, max_episode_steps):
        super().__init__()
        with self.init_scope():
            self.embed = L.EmbedID(max_episode_steps + 1, 3136)
            self.image2hidden = chainer.Sequential(
                L.Convolution2D(None, 32, 8, stride=4),
                F.relu,
                L.Convolution2D(None, 64, 4, stride=2),
                F.relu,
                L.Convolution2D(None, 64, 3, stride=1),
                functools.partial(F.reshape, shape=(-1, 3136)),
            )
            self.hidden2out = chainer.Sequential(
                L.Linear(None, 512),
                F.relu,
                L.Linear(None, n_actions),
                DiscreteActionValue,
            )

    def forward(self, x):
        image, steps = x
        h = self.image2hidden(image) * F.sigmoid(self.embed(steps))
        return self.hidden2out(h)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--outdir', type=str, default='results',
                        help='Directory path to save output files.'
                             ' If it does not exist, it will be created.')
    parser.add_argument('--seed', type=int, default=0,
                        help='Random seed [0, 2 ** 31)')
    parser.add_argument('--gpu', type=int, default=0,
                        help='GPU to use, set to -1 if no GPU.')
    parser.add_argument('--demo', action='store_true', default=False,
                        help='Evaluate the agent without training.')
    parser.add_argument('--load', type=str, default=None,
                        help='Load a saved agent from a given directory.')
    parser.add_argument('--final-exploration-steps',
                        type=int, default=5 * 10 ** 5,
                        help='Timesteps after which we stop'
                             ' annealing exploration rate')
    parser.add_argument('--final-epsilon', type=float, default=0.2,
                        help='Final value of epsilon during training.')
    parser.add_argument('--steps', type=int, default=2 * 10 ** 6,
                        help='Total number of timesteps to train the agent.')
    parser.add_argument('--replay-start-size', type=int, default=5 * 10 ** 4,
                        help='Minimum replay buffer size before'
                             ' performing gradient updates.')
    parser.add_argument('--target-update-interval',
                        type=int, default=1 * 10 ** 4,
                        help='Frequency (in timesteps) at which'
                             ' the target network is updated.')
    parser.add_argument('--eval-interval', type=int, default=10 ** 5,
                        help='Frequency (in timesteps) of evaluation phase.')
    parser.add_argument('--update-interval', type=int, default=1,
                        help='Frequency (in timesteps) of network updates.')
    parser.add_argument('--eval-n-runs', type=int, default=100,
                        help='Number of episodes used for evaluation.')
    parser.add_argument('--logging-level', type=int, default=20,
                        help='Logging level. 10:DEBUG, 20:INFO etc.')
    parser.add_argument('--render', action='store_true', default=False,
                        help='Render env states in a GUI window.')
    parser.add_argument('--lr', type=float, default=6.25e-5,
                        help='Learning rate')
    parser.add_argument('--num-envs', type=int, default=1,
                        help='Number of envs run in parallel.')
    parser.add_argument('--batch-size', type=int, default=32,
                        help='Batch size used for training.')
    parser.add_argument('--record', action='store_true', default=False,
                        help='Record videos of evaluation envs.'
                             ' --render should also be specified.')
    parser.add_argument('--gamma', type=float, default=0.99,
                        help='Discount factor.')
    args = parser.parse_args()

    import logging
    logging.basicConfig(level=args.logging_level)

    # Set a random seed used in ChainerRL.
    misc.set_random_seed(args.seed, gpus=(args.gpu,))

    # Set different random seeds for different subprocesses.
    # If seed=0 and processes=4, subprocess seeds are [0, 1, 2, 3].
    # If seed=1 and processes=4, subprocess seeds are [4, 5, 6, 7].
    process_seeds = np.arange(args.num_envs) + args.seed * args.num_envs
    assert process_seeds.max() < 2 ** 32

    args.outdir = experiments.prepare_output_dir(args, args.outdir)
    print('Output files are saved in {}'.format(args.outdir))

    max_episode_steps = 8

    def make_env(idx, test):
        from pybullet_envs.bullet.kuka_diverse_object_gym_env import KukaDiverseObjectEnv  # NOQA
        # Use different random seeds for train and test envs
        process_seed = int(process_seeds[idx])
        env_seed = 2 ** 32 - 1 - process_seed if test else process_seed
        # Set a random seed for this subprocess
        misc.set_random_seed(env_seed)
        env = KukaDiverseObjectEnv(
            isDiscrete=True,
            renders=args.render and (args.demo or not test),
            height=84,
            width=84,
            maxSteps=max_episode_steps,
            isTest=test,
        )
        # (84, 84, 3) -> (3, 84, 84)
        env = TransposeObservation(env, (2, 0, 1))
        env = ObserveElapsedSteps(env, max_episode_steps)
        env = CastAction(env, int)
        env.seed(int(env_seed))
        if test and args.record:
            assert args.render,\
                'To use --record, --render needs be specified.'
            video_dir = os.path.join(args.outdir, 'video_{}'.format(idx))
            os.mkdir(video_dir)
            env = RecordMovie(env, video_dir)
        return env

    def make_batch_env(test):
        return chainerrl.envs.MultiprocessVectorEnv(
            [functools.partial(make_env, idx, test)
                for idx in range(args.num_envs)])

    eval_env = make_batch_env(test=True)
    n_actions = eval_env.action_space.n

    q_func = GraspingQFunction(n_actions, max_episode_steps)

    # Draw the computational graph and save it in the output directory.
    fake_obs = (
        np.zeros((3, 84, 84), dtype=np.float32)[None],
        np.zeros(1, dtype=np.int32)[None],
    )
    chainerrl.misc.draw_computational_graph(
        [q_func(fake_obs)],
        os.path.join(args.outdir, 'model'))

    # Use the hyper parameters of the Nature paper
    opt = optimizers.RMSpropGraves(
        lr=args.lr, alpha=0.95, momentum=0.0, eps=1e-2)

    opt.setup(q_func)

    # Anneal beta from beta0 to 1 throughout training
    betasteps = args.steps / args.update_interval
    rbuf = replay_buffer.PrioritizedReplayBuffer(
        10 ** 6, alpha=0.6, beta0=0.4, betasteps=betasteps)

    explorer = explorers.LinearDecayEpsilonGreedy(
        1.0, args.final_epsilon,
        args.final_exploration_steps,
        lambda: np.random.randint(n_actions))

    def phi(x):
        # Feature extractor
        image, elapsed_steps = x
        # Normalize RGB values: [0, 255] -> [0, 1]
        norm_image = np.asarray(image, dtype=np.float32) / 255
        return norm_image, elapsed_steps

    agent = chainerrl.agents.DoubleDQN(
        q_func,
        opt,
        rbuf,
        gpu=args.gpu,
        gamma=args.gamma,
        explorer=explorer,
        minibatch_size=args.batch_size,
        replay_start_size=args.replay_start_size,
        target_update_interval=args.target_update_interval,
        update_interval=args.update_interval,
        batch_accumulator='sum',
        phi=phi,
    )

    if args.load:
        agent.load(args.load)

    if args.demo:
        eval_stats = experiments.eval_performance(
            env=eval_env,
            agent=agent,
            n_runs=args.eval_n_runs)
        print('n_runs: {} mean: {} median: {} stdev {}'.format(
            args.eval_n_runs, eval_stats['mean'], eval_stats['median'],
            eval_stats['stdev']))
    else:
        experiments.train_agent_batch_with_evaluation(
            agent=agent,
            env=make_batch_env(test=False),
            eval_env=eval_env,
            steps=args.steps,
            eval_n_runs=args.eval_n_runs,
            eval_interval=args.eval_interval,
            outdir=args.outdir,
            save_best_so_far_agent=False,
            log_interval=1000,
        )

if __name__ == '__main__':
    main()
<commit_msg>Fix py2 error about newint<commit_after>from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals
from __future__ import absolute_import
from builtins import *  # NOQA
from future import standard_library
standard_library.install_aliases()  # NOQA
import argparse
import functools
import os

import chainer
from chainer import functions as F
from chainer import links as L
from chainer import optimizers
import gym
import gym.spaces
import numpy as np

import chainerrl
from chainerrl.action_value import DiscreteActionValue
from chainerrl import experiments
from chainerrl import explorers
from chainerrl import misc
from chainerrl import replay_buffer

class CastAction(gym.ActionWrapper):
    """Cast actions to a given type."""

    def __init__(self, env, type_):
        super().__init__(env)
        self.type_ = type_

    def _action(self, action):
        return self.type_(action)

class TransposeObservation(gym.ObservationWrapper):
    """Transpose observations."""

    def __init__(self, env, axes):
        super().__init__(env)
        self._axes = axes

    def _observation(self, observation):
        return observation.transpose(*self._axes)

class ObserveElapsedSteps(gym.Wrapper):
    """Observe the number of elapsed steps in an episode.

    A new observation will be a tuple of an original observation and an integer
    that is equal to the elapsed steps in an episode.
    """

    def __init__(self, env, max_steps):
        super().__init__(env)
        self._max_steps = max_steps
        self._elapsed_steps = 0
        self.observation_space = gym.spaces.Tuple((
            env.observation_space,
            gym.spaces.Discrete(self._max_steps + 1),
        ))

    def reset(self):
        self._elapsed_steps = 0
        return self.env.reset(), self._elapsed_steps

    def _step(self, action):
        observation, reward, done, info = self.env.step(action)
        self._elapsed_steps += 1
        assert self._elapsed_steps <= self._max_steps
        return (observation, self._elapsed_steps), reward, done, info

class RecordMovie(gym.Wrapper):
    """Record MP4 videos using pybullet's logging API."""

    def __init__(self, env, dirname):
        super().__init__(env)
        self._episode_idx = -1
        self._dirname = dirname

    def reset(self):
        obs = self.env.reset()
        self._episode_idx += 1
        import pybullet
        pybullet.startStateLogging(
            pybullet.STATE_LOGGING_VIDEO_MP4,
            os.path.join(self._dirname, '{}.mp4'.format(self._episode_idx)))
        return obs

class GraspingQFunction(chainer.Chain):
    """Q-function model for the grasping env.

    This model takes an 84x84 2D image and an integer that indicates the
    number of elapsed steps in an episode as input and outputs action values.
    """

    def __init__(self, n_actions, max_episode_steps):
        super().__init__()
        with self.init_scope():
            self.embed = L.EmbedID(max_episode_steps + 1, 3136)
            self.image2hidden = chainer.Sequential(
                L.Convolution2D(None, 32, 8, stride=4),
                F.relu,
                L.Convolution2D(None, 64, 4, stride=2),
                F.relu,
                L.Convolution2D(None, 64, 3, stride=1),
                functools.partial(F.reshape, shape=(-1, 3136)),
            )
            self.hidden2out = chainer.Sequential(
                L.Linear(None, 512),
                F.relu,
                L.Linear(None, n_actions),
                DiscreteActionValue,
            )

    def forward(self, x):
        image, steps = x
        h = self.image2hidden(image) * F.sigmoid(self.embed(steps))
        return self.hidden2out(h)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--outdir', type=str, default='results',
                        help='Directory path to save output files.'
                             ' If it does not exist, it will be created.')
    parser.add_argument('--seed', type=int, default=0,
                        help='Random seed [0, 2 ** 31)')
    parser.add_argument('--gpu', type=int, default=0,
                        help='GPU to use, set to -1 if no GPU.')
    parser.add_argument('--demo', action='store_true', default=False,
                        help='Evaluate the agent without training.')
    parser.add_argument('--load', type=str, default=None,
                        help='Load a saved agent from a given directory.')
    parser.add_argument('--final-exploration-steps',
                        type=int, default=5 * 10 ** 5,
                        help='Timesteps after which we stop'
                             ' annealing exploration rate')
    parser.add_argument('--final-epsilon', type=float, default=0.2,
                        help='Final value of epsilon during training.')
    parser.add_argument('--steps', type=int, default=2 * 10 ** 6,
                        help='Total number of timesteps to train the agent.')
    parser.add_argument('--replay-start-size', type=int, default=5 * 10 ** 4,
                        help='Minimum replay buffer size before'
                             ' performing gradient updates.')
    parser.add_argument('--target-update-interval',
                        type=int, default=1 * 10 ** 4,
                        help='Frequency (in timesteps) at which'
                             ' the target network is updated.')
    parser.add_argument('--eval-interval', type=int, default=10 ** 5,
                        help='Frequency (in timesteps) of evaluation phase.')
    parser.add_argument('--update-interval', type=int, default=1,
                        help='Frequency (in timesteps) of network updates.')
    parser.add_argument('--eval-n-runs', type=int, default=100,
                        help='Number of episodes used for evaluation.')
    parser.add_argument('--logging-level', type=int, default=20,
                        help='Logging level. 10:DEBUG, 20:INFO etc.')
    parser.add_argument('--render', action='store_true', default=False,
                        help='Render env states in a GUI window.')
    parser.add_argument('--lr', type=float, default=6.25e-5,
                        help='Learning rate')
    parser.add_argument('--num-envs', type=int, default=1,
                        help='Number of envs run in parallel.')
    parser.add_argument('--batch-size', type=int, default=32,
                        help='Batch size used for training.')
    parser.add_argument('--record', action='store_true', default=False,
                        help='Record videos of evaluation envs.'
                             ' --render should also be specified.')
    parser.add_argument('--gamma', type=float, default=0.99,
                        help='Discount factor.')
    args = parser.parse_args()

    import logging
    logging.basicConfig(level=args.logging_level)

    # Set a random seed used in ChainerRL.
    misc.set_random_seed(args.seed, gpus=(args.gpu,))

    # Set different random seeds for different subprocesses.
    # If seed=0 and processes=4, subprocess seeds are [0, 1, 2, 3].
    # If seed=1 and processes=4, subprocess seeds are [4, 5, 6, 7].
    process_seeds = np.arange(args.num_envs) + args.seed * args.num_envs
    assert process_seeds.max() < 2 ** 32

    args.outdir = experiments.prepare_output_dir(args, args.outdir)
    print('Output files are saved in {}'.format(args.outdir))

    max_episode_steps = 8

    def make_env(idx, test):
        from pybullet_envs.bullet.kuka_diverse_object_gym_env import KukaDiverseObjectEnv  # NOQA
        # Use different random seeds for train and test envs
        process_seed = int(process_seeds[idx])
        env_seed = 2 ** 32 - 1 - process_seed if test else process_seed
        # Set a random seed for this subprocess
        misc.set_random_seed(env_seed)
        env = KukaDiverseObjectEnv(
            isDiscrete=True,
            renders=args.render and (args.demo or not test),
            height=84,
            width=84,
            maxSteps=max_episode_steps,
            isTest=test,
        )
        # (84, 84, 3) -> (3, 84, 84)
        env = TransposeObservation(env, (2, 0, 1))
        env = ObserveElapsedSteps(env, max_episode_steps)
        # KukaDiverseObjectEnv internally asserts int actions and does not
        # accept python-future's newint.
        import __builtin__
        env = CastAction(env, __builtin__.int)
        env.seed(int(env_seed))
        if test and args.record:
            assert args.render,\
                'To use --record, --render needs be specified.'
            video_dir = os.path.join(args.outdir, 'video_{}'.format(idx))
            os.mkdir(video_dir)
            env = RecordMovie(env, video_dir)
        return env

    def make_batch_env(test):
        return chainerrl.envs.MultiprocessVectorEnv(
            [functools.partial(make_env, idx, test)
                for idx in range(args.num_envs)])

    eval_env = make_batch_env(test=True)
    n_actions = eval_env.action_space.n

    q_func = GraspingQFunction(n_actions, max_episode_steps)

    # Draw the computational graph and save it in the output directory.
    fake_obs = (
        np.zeros((3, 84, 84), dtype=np.float32)[None],
        np.zeros(1, dtype=np.int32)[None],
    )
    chainerrl.misc.draw_computational_graph(
        [q_func(fake_obs)],
        os.path.join(args.outdir, 'model'))

    # Use the hyper parameters of the Nature paper
    opt = optimizers.RMSpropGraves(
        lr=args.lr, alpha=0.95, momentum=0.0, eps=1e-2)

    opt.setup(q_func)

    # Anneal beta from beta0 to 1 throughout training
    betasteps = args.steps / args.update_interval
    rbuf = replay_buffer.PrioritizedReplayBuffer(
        10 ** 6, alpha=0.6, beta0=0.4, betasteps=betasteps)

    explorer = explorers.LinearDecayEpsilonGreedy(
        1.0, args.final_epsilon,
        args.final_exploration_steps,
        lambda: np.random.randint(n_actions))

    def phi(x):
        # Feature extractor
        image, elapsed_steps = x
        # Normalize RGB values: [0, 255] -> [0, 1]
        norm_image = np.asarray(image, dtype=np.float32) / 255
        return norm_image, elapsed_steps

    agent = chainerrl.agents.DoubleDQN(
        q_func,
        opt,
        rbuf,
        gpu=args.gpu,
        gamma=args.gamma,
        explorer=explorer,
        minibatch_size=args.batch_size,
        replay_start_size=args.replay_start_size,
        target_update_interval=args.target_update_interval,
        update_interval=args.update_interval,
        batch_accumulator='sum',
        phi=phi,
    )

    if args.load:
        agent.load(args.load)

    if args.demo:
        eval_stats = experiments.eval_performance(
            env=eval_env,
            agent=agent,
            n_runs=args.eval_n_runs)
        print('n_runs: {} mean: {} median: {} stdev {}'.format(
            args.eval_n_runs, eval_stats['mean'], eval_stats['median'],
            eval_stats['stdev']))
    else:
        experiments.train_agent_batch_with_evaluation(
            agent=agent,
            env=make_batch_env(test=False),
            eval_env=eval_env,
            steps=args.steps,
            eval_n_runs=args.eval_n_runs,
            eval_interval=args.eval_interval,
            outdir=args.outdir,
            save_best_so_far_agent=False,
            log_interval=1000,
        )

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Fab file to help with managing project.  For docs on Fab file, please see: http://docs.fabfile.org/
"""
import sys
import os
import warnings
import json
import re
from fabric.api import *

"""
Base configuration
"""
env.project_name = 'minnpost-basemaps'
env.pg_host = 'localhost'
env.pg_dbname = 'minnpost_fec'
env.pg_user = 'postgres'
env.pg_pass = ''
env.labels = None
env.tile_scheme = 'tms'
env.tile_template = '{z}/{x}/{y}'

# Tilemill paths.  For Ubuntu
if os.path.exists('/usr/share/tilemill'):
  env.tilemill_path = '/usr/share/tilemill'
  env.tilemill_projects = '/usr/share/mapbox/project'
  env.node_path = '/usr/bin/node'
# OSX
else:
  env.tilemill_path = '/Applications/TileMill.app/Contents/Resources'
  env.tilemill_projects = '~/Documents/MapBox/project'
  env.node_path = '%(tilemill_path)s/node' % env

"""
Environments
"""
def production():
  """
  Work on production environment
  """
  env.settings = 'production'
  #env.s3_buckets = ['a.tiles.minnpost', 'b.tiles.minnpost', 'c.tiles.minnpost', 'd.tiles.minnpost']
  env.s3_buckets = ['a.tiles.minnpost']
  env.s3_template = 'http://{s}.tiles.minnpost.com'
  env.acl = 'acl-public'

def staging():
  """
  Work on staging environment
  """
  env.settings = 'staging'
  env.s3_buckets = ['testing.tiles.minnpost']
  env.s3_template = 'http://testing.tiles.minnpost.s3.amazonaws.com'
  env.acl = 'acl-public'

def map(name):
  """
  Select map to work on.
  """
  env.map = name

def tile_scheme(scheme='tms'):
  """
  Change tilescheme.
  """
  env.tile_scheme = scheme

def deploy_to_s3(concurrency):
  """
  Deploy tiles to S3.
  """
  require('settings', provided_by=[production, staging])
  require('map', provided_by=[map])
  env.concurrency = concurrency
  _create_map_suffix()

  # Deploy to many buckets (multi-dns-head mode)
  for bucket in env.s3_buckets:
    env.s3_bucket = bucket
    local('ivs3 %(map)s/tiles %(s3_bucket)s/%(project_name)s/%(map)s%(map_suffix)s --%(acl)s -c %(concurrency)s' % env)

def export_deploy(concurrency=32, minzoom=None, maxzoom=None):
  """
  Deploy a map. Optionally takes a concurrency parameter indicating how many files to upload simultaneously.
  """
  require('settings', provided_by=[production, staging])
  require('map', provided_by=[map])

  create_exports()
  cleanup_exports()
  generate_mbtile(minzoom, maxzoom)
  generate_tiles_from_mbtile()
  generate_tilejson()
  deploy_to_s3(concurrency)
  reset_labels()

def link_caches():
  """
  Link local map cache to Mapbox cache to speed up mapnik conversion.
  """
  require('map', provided_by=[map])

  env.base_path = os.getcwd()
  exists = os.path.exists('%(base_path)s/%(map)s/cache' % env)
  if exists:
    print "There already a linked cache directory."
  else:
    env.tilemill_cache = os.path.expanduser('%(tilemill_projects)s/../cache/' % env)
    local(('ln -s %(tilemill_cache)s %(base_path)s/%(map)s/cache') % env)

def carto_to_mapnik():
  """
  Convert carto styles to mapnik configuration.  This should be able to
  be done with the Tilemill export API.
  """
  require('map', provided_by=[map])
  link_caches()
  local('%(tilemill_path)s/node_modules/carto/bin/carto %(map)s/project.mml > %(map)s/%(map)s.xml' % env)

def generate_mbtile(minzoom=None, maxzoom=None):
  """
  Generate MBtile.
  """
  require('map', provided_by=[map])

  # Read data from project mml
  with open('%(map)s/project.mml' % env, 'r') as f:
    config = json.load(f)

    # Define config values
    env.minzoom = config['minzoom'] if minzoom == None else minzoom
    env.maxzoom = config['maxzoom'] if maxzoom == None else maxzoom
    env.bbox = '%f,%f,%f,%f' % (config['bounds'][0], config['bounds'][1], config['bounds'][2], config['bounds'][3])

    # Export
    local('%(tilemill_path)s/node %(tilemill_path)s/index.js export --format=mbtiles --minzoom=%(minzoom)s --maxzoom=%(maxzoom)s --bbox=%(bbox)s %(map)s %(map)s/exports/%(map)s.mbtiles' % env)

def generate_tiles_from_mbtile():
  """
  Generate MBtile.
  """
  require('map', provided_by=[map])
  read_project()

  exists = os.path.exists('%(map)s/exports/%(map)s.mbtiles' % env)
  if exists:
    with settings(warn_only=True):
      local('rm -rf %(map)s/tiles-tmp' % env)
      local('rm -rf %(map)s/tiles' % env)
      local('mb-util --scheme=%(tile_scheme)s %(map)s/exports/%(map)s.mbtiles %(map)s/tiles-tmp' % env)
      local('mv "%(map)s/tiles-tmp/%(map_version)s/%(map_title)s" %(map)s/tiles' % env)
      local('mv %(map)s/tiles-tmp/metadata.json %(map)s/tiles/metadata.json' % env)
  else:
    print 'No MBTile file found in exports.'

def generate_tilejson():
  """
  Generate valid tilejson file.
  """
  require('settings', provided_by=[production, staging])
  require('map', provided_by=[map])
  _create_map_suffix()

  # Utilize project data
  with open('%(map)s/project.mml' % env, 'r') as f:
    config = json.load(f)
    tilejson = {}

    # Base values
    tilejson['scheme'] = env.tile_scheme
    tilejson['tilejson'] = '2.0.0'
    tilejson['id'] = env.map

    # Attempt to get values from config
    try:
      tilejson['name'] = config['name'] if config.has_key('name') else ''
      tilejson['description'] = config['description'] if config.has_key('description') else ''
      tilejson['version'] = config['version'] if config.has_key('version') else '1.0.0'
      tilejson['attribution'] = config['attribution'] if config.has_key('attribution') else ''
      tilejson['legend'] = config['legend'] if config.has_key('legend') else ''
      tilejson['minzoom'] = config['minzoom'] if config.has_key('minzoom') else 0
      tilejson['maxzoom'] = config['maxzoom'] if config.has_key('maxzoom') else 22
      tilejson['bounds'] = config['bounds'] if config.has_key('bounds') else [-180, -90, 180, 90]
      tilejson['center'] = config['center'] if config.has_key('center') else null
    except KeyError:
      print 'Key error'

    # Template is in the metadata.json file
    with open('%(map)s/tiles/metadata.json' % env, 'r') as j:
      metadata = json.load(j)
      try:
        tilejson['template'] = metadata['template']
      except KeyError:
        tilejson['template'] = ''

    # Figure out template
    tilejson['grids'] = []
    tilejson['tiles'] = []
    for bucket in env.s3_buckets:
      env.s3_bucket = bucket
      tilejson['grids'].append('http://%(s3_bucket)s.s3.amazonaws.com/%(project_name)s/%(map)s%(map_suffix)s/%(tile_template)s.grid.json' % env)
      tilejson['tiles'].append('http://%(s3_bucket)s.s3.amazonaws.com/%(project_name)s/%(map)s%(map_suffix)s/%(tile_template)s.png' % env)

    # Write regular and jsonp tilejson files
    tilejson_file = open('%(map)s/tiles/tilejson.json' % env, 'w')
    tilejson_file.write(json.dumps(tilejson, sort_keys = True, indent = 2))
    tilejson_file.close()
    tilejsonp_file = open('%(map)s/tiles/tilejson.jsonp' % env, 'w')
    tilejsonp_file.write('grid(%s)' % json.dumps(tilejson, sort_keys = True, indent = 2))
    tilejsonp_file.close()

def generate_tiles_mapnik(process_count, minzoom=None, maxzoom=None):
  """
  Render tile from mapnik configuration
  """
  env.process_count = process_count

  # Read data from project mml
  with open('%(map)s/project.mml' % env, 'r') as f:
    config = json.load(f)

    # Define config values
    env.minzoom = config['minzoom'] if minzoom == None else minzoom
    env.maxzoom = config['maxzoom'] if maxzoom == None else maxzoom
    env.minlon = config['bounds'][0]
    env.minlat = config['bounds'][1]
    env.maxlon = config['bounds'][2]
    env.maxlat = config['bounds'][3]

    # Cleanup tiles
    local('rm -rf %(map)s/tiles/*' % env)

    # Render tiles
    command = 'ivtile %(map)s/%(map)s.xml %(map)s/tiles %(maxlat)s %(minlon)s %(minlat)s %(maxlon)s %(minzoom)s %(maxzoom)s -p %(process_count)s'
    if 'buffer' in env:
      command += ' -b %(buffer)s'
    local(command % env)

def no_labels():
  """
  Updates tilemill project to no use labels
  """
  require('map', provided_by=[map])
  env.labels = False

  # Create a backup
  exists = os.path.exists('%(map)s/project.mml.orig' % env)
  if exists != True:
    local('cp %(map)s/project.mml %(map)s/project.mml.orig' % env);

  # From the original, load the json, find any label style
  # and remove.
  overwrite = open('%(map)s/project.mml' % env, 'w')
  with open('%(map)s/project.mml.orig' % env, 'r') as f:
    mml = json.load(f)

    # Process styles
    styles = []
    for index, item in enumerate(mml['Stylesheet']):
      if item != 'labels.mss':
        styles.append(item)
    mml['Stylesheet'] = styles

    overwrite.write(json.dumps(mml, sort_keys = True, indent = 2))
    overwrite.close()

def only_labels():
  """
  Updates tilemill project to use only labels
  """
  require('map', provided_by=[map])
  env.labels = True

  # Create a backup
  exists = os.path.exists('%(map)s/project.mml.orig' % env)
  if exists != True:
    local('cp %(map)s/project.mml %(map)s/project.mml.orig' % env);

  # From the original, load the json, keep palette and labels
  overwrite = open('%(map)s/project.mml' % env, 'w')
  with open('%(map)s/project.mml.orig' % env, 'r') as f:
    mml = json.load(f)

    # Process styles
    styles = []
    for index, item in enumerate(mml['Stylesheet']):
      if item == 'labels.mss' or item == 'palette.mss':
        styles.append(item)
    mml['Stylesheet'] = styles

    overwrite.write(json.dumps(mml, sort_keys = True, indent = 2))
    overwrite.close()

def reset_labels():
  """
  Resets any label changes process
  """
  require('map', provided_by=[map])

  # Check for orig
  exists = os.path.exists('%(map)s/project.mml.orig' % env)
  if exists:
    local('rm -f %(map)s/project.mml' % env);
    local('mv %(map)s/project.mml.orig %(map)s/project.mml' % env);
  else:
    print 'No label processing to reset.'

def _create_map_suffix():
  """
  Creates map suffix for deploying
  """
  if env.labels == None:
    env.map_suffix = ''
  if env.labels == True:
    env.map_suffix = '-labels'
  if env.labels == False:
    env.map_suffix = '-no-labels'

def read_project():
  """
  Get data from the TileMill project, to be used in other
  commands.
  """
  require('map', provided_by=[map])

  # Read JSON from file
  with open('%(map)s/project.mml' % env, 'r') as f:
    mml = json.load(f)
    env.map_title = mml['name']

    # Version is not always defined
    try:
      env.map_version = mml['version']
    except KeyError:
      env.map_version = '1.0.0'

def create_exports():
  """
  Create export directories
  """
  require('map', provided_by=[map])
  local('mkdir -p %(map)s/tiles' % env)
  local('mkdir -p %(map)s/exports' % env)

def cleanup_exports():
  """
  Cleanup export directories
  """
  require('map', provided_by=[map])
  local('rm -rf %(map)s/tiles-tmp' % env)
  local('rm -rf %(map)s/tiles/*' % env)
  local('rm -rf %(map)s/exports/*' % env)

def link():
  """
  Link a map into the MapBox directory
  """
  require('map', provided_by=[map])

  exists = os.path.exists(os.path.expanduser('%(tilemill_projects)s/%(map)s' % env))
  if exists:
    print "A directory with that name already exists in your MapBox projects."
  else:
    env.base_path = os.getcwd()
    local(('ln -s %(base_path)s/%(map)s/ %(tilemill_projects)s/%(map)s') % env)

def unlink():
  """
  unLink a map into the MapBox directory
  """
  require('map', provided_by=[map])

  exists = os.path.exists(os.path.expanduser('%(tilemill_projects)s/%(map)s' % env))
  if exists:
    local(('unlink %(tilemill_projects)s/%(map)s') % env)
  else:
    print "There is no directpry with that name in your MapBox projects."

def clone(name):
  """
  Clone a map to work on.
  """
  require('map', provided_by=[map])
  env.clone = name

  exists = os.path.exists('%(clone)s' % env)
  if exists:
    print "A directory with that name already exists in this project."
  else:
    local('cp -r %(map)s %(clone)s' % env)

    # Now link it
    env.map = env.clone
    link()

def remove_from_s3():
  """
  Remove map from S3
  """
  require('settings', provided_by=[production, staging])
  require('map', provided_by=[map])

  with settings(warn_only=True):
    for bucket in env.s3_buckets:
      env.s3_bucket = bucket
      local('s3cmd del --recursive s3://%(s3_bucket)s/%(project_name)s/%(map)s' % env)<commit_msg>Moving to xyz scheme by default.<commit_after>#!/usr/bin/env python
"""
Fab file to help with managing project.  For docs on Fab file, please see: http://docs.fabfile.org/
"""
import sys
import os
import warnings
import json
import re
from fabric.api import *

"""
Base configuration
"""
env.project_name = 'minnpost-basemaps'
env.pg_host = 'localhost'
env.pg_dbname = 'minnpost_fec'
env.pg_user = 'postgres'
env.pg_pass = ''
env.labels = None
env.tile_scheme = 'xyz'
env.tile_template = '{z}/{x}/{y}'

# Tilemill paths.  For Ubuntu
if os.path.exists('/usr/share/tilemill'):
  env.tilemill_path = '/usr/share/tilemill'
  env.tilemill_projects = '/usr/share/mapbox/project'
  env.node_path = '/usr/bin/node'
# OSX
else:
  env.tilemill_path = '/Applications/TileMill.app/Contents/Resources'
  env.tilemill_projects = '~/Documents/MapBox/project'
  env.node_path = '%(tilemill_path)s/node' % env

"""
Environments
"""
def production():
  """
  Work on production environment
  """
  env.settings = 'production'
  #env.s3_buckets = ['a.tiles.minnpost', 'b.tiles.minnpost', 'c.tiles.minnpost', 'd.tiles.minnpost']
  env.s3_buckets = ['a.tiles.minnpost']
  env.s3_template = 'http://{s}.tiles.minnpost.com'
  env.acl = 'acl-public'

def staging():
  """
  Work on staging environment
  """
  env.settings = 'staging'
  env.s3_buckets = ['testing.tiles.minnpost']
  env.s3_template = 'http://testing.tiles.minnpost.s3.amazonaws.com'
  env.acl = 'acl-public'

def map(name):
  """
  Select map to work on.
  """
  env.map = name

def tile_scheme(scheme='xyz'):
  """
  Change tilescheme.
  """
  env.tile_scheme = scheme

def deploy_to_s3(concurrency):
  """
  Deploy tiles to S3.
  """
  require('settings', provided_by=[production, staging])
  require('map', provided_by=[map])
  env.concurrency = concurrency
  _create_map_suffix()

  # Deploy to many buckets (multi-dns-head mode)
  for bucket in env.s3_buckets:
    env.s3_bucket = bucket
    local('ivs3 %(map)s/tiles %(s3_bucket)s/%(project_name)s/%(map)s%(map_suffix)s --%(acl)s -c %(concurrency)s' % env)

def export_deploy(concurrency=32, minzoom=None, maxzoom=None):
  """
  Deploy a map. Optionally takes a concurrency parameter indicating how many files to upload simultaneously.
  """
  require('settings', provided_by=[production, staging])
  require('map', provided_by=[map])

  create_exports()
  cleanup_exports()
  generate_mbtile(minzoom, maxzoom)
  generate_tiles_from_mbtile()
  generate_tilejson()
  deploy_to_s3(concurrency)
  reset_labels()

def link_caches():
  """
  Link local map cache to Mapbox cache to speed up mapnik conversion.
  """
  require('map', provided_by=[map])

  env.base_path = os.getcwd()
  exists = os.path.exists('%(base_path)s/%(map)s/cache' % env)
  if exists:
    print "There already a linked cache directory."
  else:
    env.tilemill_cache = os.path.expanduser('%(tilemill_projects)s/../cache/' % env)
    local(('ln -s %(tilemill_cache)s %(base_path)s/%(map)s/cache') % env)

def carto_to_mapnik():
  """
  Convert carto styles to mapnik configuration.  This should be able to
  be done with the Tilemill export API.
  """
  require('map', provided_by=[map])
  link_caches()
  local('%(tilemill_path)s/node_modules/carto/bin/carto %(map)s/project.mml > %(map)s/%(map)s.xml' % env)

def generate_mbtile(minzoom=None, maxzoom=None):
  """
  Generate MBtile.
  """
  require('map', provided_by=[map])

  # Read data from project mml
  with open('%(map)s/project.mml' % env, 'r') as f:
    config = json.load(f)

    # Define config values
    env.minzoom = config['minzoom'] if minzoom == None else minzoom
    env.maxzoom = config['maxzoom'] if maxzoom == None else maxzoom
    env.bbox = '%f,%f,%f,%f' % (config['bounds'][0], config['bounds'][1], config['bounds'][2], config['bounds'][3])

    # Export
    local('%(tilemill_path)s/node %(tilemill_path)s/index.js export --format=mbtiles --minzoom=%(minzoom)s --maxzoom=%(maxzoom)s --bbox=%(bbox)s %(map)s %(map)s/exports/%(map)s.mbtiles' % env)

def generate_tiles_from_mbtile():
  """
  Generate MBtile.
  """
  require('map', provided_by=[map])
  read_project()

  exists = os.path.exists('%(map)s/exports/%(map)s.mbtiles' % env)
  if exists:
    with settings(warn_only=True):
      local('rm -rf %(map)s/tiles-tmp' % env)
      local('rm -rf %(map)s/tiles' % env)
      local('mb-util --scheme=%(tile_scheme)s %(map)s/exports/%(map)s.mbtiles %(map)s/tiles-tmp' % env)
      local('mv "%(map)s/tiles-tmp/%(map_version)s/%(map_title)s" %(map)s/tiles' % env)
      local('mv %(map)s/tiles-tmp/metadata.json %(map)s/tiles/metadata.json' % env)
  else:
    print 'No MBTile file found in exports.'

def generate_tilejson():
  """
  Generate valid tilejson file.  This should be able to be accomplished
  with the api, such as http://localhost:20009/api/Project/:id
  """
  require('settings', provided_by=[production, staging])
  require('map', provided_by=[map])
  _create_map_suffix()

  # Utilize project data
  with open('%(map)s/project.mml' % env, 'r') as f:
    config = json.load(f)
    tilejson = {}

    # Base values
    tilejson['scheme'] = env.tile_scheme
    tilejson['tilejson'] = '2.0.0'
    tilejson['id'] = env.map

    # Attempt to get values from config
    try:
      tilejson['name'] = config['name'] if config.has_key('name') else ''
      tilejson['description'] = config['description'] if config.has_key('description') else ''
      tilejson['version'] = config['version'] if config.has_key('version') else '1.0.0'
      tilejson['attribution'] = config['attribution'] if config.has_key('attribution') else ''
      tilejson['legend'] = config['legend'] if config.has_key('legend') else ''
      tilejson['minzoom'] = config['minzoom'] if config.has_key('minzoom') else 0
      tilejson['maxzoom'] = config['maxzoom'] if config.has_key('maxzoom') else 22
      tilejson['bounds'] = config['bounds'] if config.has_key('bounds') else [-180, -90, 180, 90]
      tilejson['center'] = config['center'] if config.has_key('center') else null
    except KeyError:
      print 'Key error'

    # Template is in the metadata.json file
    with open('%(map)s/tiles/metadata.json' % env, 'r') as j:
      metadata = json.load(j)
      try:
        tilejson['template'] = metadata['template']
      except KeyError:
        tilejson['template'] = ''

    # Figure out template
    tilejson['grids'] = []
    tilejson['tiles'] = []
    for bucket in env.s3_buckets:
      env.s3_bucket = bucket
      tilejson['grids'].append('http://%(s3_bucket)s.s3.amazonaws.com/%(project_name)s/%(map)s%(map_suffix)s/%(tile_template)s.grid.json' % env)
      tilejson['tiles'].append('http://%(s3_bucket)s.s3.amazonaws.com/%(project_name)s/%(map)s%(map_suffix)s/%(tile_template)s.png' % env)

    # Write regular and jsonp tilejson files
    tilejson_file = open('%(map)s/tiles/tilejson.json' % env, 'w')
    tilejson_file.write(json.dumps(tilejson, sort_keys = True, indent = 2))
    tilejson_file.close()
    tilejsonp_file = open('%(map)s/tiles/tilejson.jsonp' % env, 'w')
    tilejsonp_file.write('grid(%s)' % json.dumps(tilejson, sort_keys = True, indent = 2))
    tilejsonp_file.close()

def generate_tiles_mapnik(process_count, minzoom=None, maxzoom=None):
  """
  Render tile from mapnik configuration
  """
  env.process_count = process_count

  # Read data from project mml
  with open('%(map)s/project.mml' % env, 'r') as f:
    config = json.load(f)

    # Define config values
    env.minzoom = config['minzoom'] if minzoom == None else minzoom
    env.maxzoom = config['maxzoom'] if maxzoom == None else maxzoom
    env.minlon = config['bounds'][0]
    env.minlat = config['bounds'][1]
    env.maxlon = config['bounds'][2]
    env.maxlat = config['bounds'][3]

    # Cleanup tiles
    local('rm -rf %(map)s/tiles/*' % env)

    # Render tiles
    command = 'ivtile %(map)s/%(map)s.xml %(map)s/tiles %(maxlat)s %(minlon)s %(minlat)s %(maxlon)s %(minzoom)s %(maxzoom)s -p %(process_count)s'
    if 'buffer' in env:
      command += ' -b %(buffer)s'
    local(command % env)

def no_labels():
  """
  Updates tilemill project to no use labels
  """
  require('map', provided_by=[map])
  env.labels = False

  # Create a backup
  exists = os.path.exists('%(map)s/project.mml.orig' % env)
  if exists != True:
    local('cp %(map)s/project.mml %(map)s/project.mml.orig' % env);

  # From the original, load the json, find any label style
  # and remove.
  overwrite = open('%(map)s/project.mml' % env, 'w')
  with open('%(map)s/project.mml.orig' % env, 'r') as f:
    mml = json.load(f)

    # Process styles
    styles = []
    for index, item in enumerate(mml['Stylesheet']):
      if item != 'labels.mss':
        styles.append(item)
    mml['Stylesheet'] = styles

    overwrite.write(json.dumps(mml, sort_keys = True, indent = 2))
    overwrite.close()

def only_labels():
  """
  Updates tilemill project to use only labels
  """
  require('map', provided_by=[map])
  env.labels = True

  # Create a backup
  exists = os.path.exists('%(map)s/project.mml.orig' % env)
  if exists != True:
    local('cp %(map)s/project.mml %(map)s/project.mml.orig' % env);

  # From the original, load the json, keep palette and labels
  overwrite = open('%(map)s/project.mml' % env, 'w')
  with open('%(map)s/project.mml.orig' % env, 'r') as f:
    mml = json.load(f)

    # Process styles
    styles = []
    for index, item in enumerate(mml['Stylesheet']):
      if item == 'labels.mss' or item == 'palette.mss':
        styles.append(item)
    mml['Stylesheet'] = styles

    overwrite.write(json.dumps(mml, sort_keys = True, indent = 2))
    overwrite.close()

def reset_labels():
  """
  Resets any label changes process
  """
  require('map', provided_by=[map])

  # Check for orig
  exists = os.path.exists('%(map)s/project.mml.orig' % env)
  if exists:
    local('rm -f %(map)s/project.mml' % env);
    local('mv %(map)s/project.mml.orig %(map)s/project.mml' % env);
  else:
    print 'No label processing to reset.'

def _create_map_suffix():
  """
  Creates map suffix for deploying
  """
  if env.labels == None:
    env.map_suffix = ''
  if env.labels == True:
    env.map_suffix = '-labels'
  if env.labels == False:
    env.map_suffix = '-no-labels'

def read_project():
  """
  Get data from the TileMill project, to be used in other
  commands.
  """
  require('map', provided_by=[map])

  # Read JSON from file
  with open('%(map)s/project.mml' % env, 'r') as f:
    mml = json.load(f)
    env.map_title = mml['name']

    # Version is not always defined
    try:
      env.map_version = mml['version']
    except KeyError:
      env.map_version = '1.0.0'

def create_exports():
  """
  Create export directories
  """
  require('map', provided_by=[map])
  local('mkdir -p %(map)s/tiles' % env)
  local('mkdir -p %(map)s/exports' % env)

def cleanup_exports():
  """
  Cleanup export directories
  """
  require('map', provided_by=[map])
  local('rm -rf %(map)s/tiles-tmp' % env)
  local('rm -rf %(map)s/tiles/*' % env)
  local('rm -rf %(map)s/exports/*' % env)

def link():
  """
  Link a map into the MapBox directory
  """
  require('map', provided_by=[map])

  exists = os.path.exists(os.path.expanduser('%(tilemill_projects)s/%(map)s' % env))
  if exists:
    print "A directory with that name already exists in your MapBox projects."
  else:
    env.base_path = os.getcwd()
    local(('ln -s %(base_path)s/%(map)s/ %(tilemill_projects)s/%(map)s') % env)

def unlink():
  """
  unLink a map into the MapBox directory
  """
  require('map', provided_by=[map])

  exists = os.path.exists(os.path.expanduser('%(tilemill_projects)s/%(map)s' % env))
  if exists:
    local(('unlink %(tilemill_projects)s/%(map)s') % env)
  else:
    print "There is no directpry with that name in your MapBox projects."

def clone(name):
  """
  Clone a map to work on.
  """
  require('map', provided_by=[map])
  env.clone = name

  exists = os.path.exists('%(clone)s' % env)
  if exists:
    print "A directory with that name already exists in this project."
  else:
    local('cp -r %(map)s %(clone)s' % env)

    # Now link it
    env.map = env.clone
    link()

def remove_from_s3():
  """
  Remove map from S3
  """
  require('settings', provided_by=[production, staging])
  require('map', provided_by=[map])

  with settings(warn_only=True):
    for bucket in env.s3_buckets:
      env.s3_bucket = bucket
      local('s3cmd del --recursive s3://%(s3_bucket)s/%(project_name)s/%(map)s' % env)<|endoftext|>