<commit_before>#
# Copyright (c) 2014, Arista Networks, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#   Redistributions of source code must retain the above copyright notice,
#   this list of conditions and the following disclaimer.
#
#   Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
#
#   Neither the name of Arista Networks nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL ARISTA NETWORKS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
import os
import unittest

import sys
sys.path.append(os.path.join(os.path.dirname(__file__), '../lib'))

from testlib import random_int, random_string, get_fixture

import pyeapi.client
import pyeapi.eapilib

class TestClient(unittest.TestCase):

    def setUp(self):
        pyeapi.client.load_config(filename=get_fixture('dut.conf'))
        config = pyeapi.client.config

        self.duts = list()
        for name in config.sections():
            if name.startswith('connection:') and 'localhost' not in name:
                name = name.split(':')[1]
                dut = pyeapi.client.connect_to(name)
                self.duts.append(dut)
                if dut._enablepwd is not None:
                    # If enable password defined for dut, set the
                    # enable password on the dut and clear it on tearDown
                    dut.config("enable secret %s" % dut._enablepwd)

    def test_enable_single_command(self):
        for dut in self.duts:
            result = dut.run_commands('show version')
            self.assertIsInstance(result, list, 'dut=%s' % dut)
            self.assertEqual(len(result), 1, 'dut=%s' % dut)

    def test_no_enable_single_command(self):
        for dut in self.duts:
            result = dut.run_commands('show version', 'json', send_enable=False)
            self.assertIsInstance(result, list, 'dut=%s' % dut)
            self.assertEqual(len(result), 1, 'dut=%s' % dut)

    def test_no_enable_single_command_no_auth(self):
        for dut in self.duts:
            with self.assertRaises(pyeapi.eapilib.CommandError):
                dut.run_commands(['disable', 'show running-config'], 'json', send_enable=False)

    def test_enable_multiple_commands(self):
        for dut in self.duts:
            commands = list()
            for i in range(1, random_int(10, 200)):
                commands.append('show version')
            result = dut.run_commands(commands[:])
            self.assertIsInstance(result, list, 'dut=%s' % dut)
            self.assertEqual(len(result), len(commands), 'dut=%s' % dut)

    def test_config_single_command(self):
        for dut in self.duts:
            hostname = 'hostname %s' % random_string(5, 50)
            result = dut.config(hostname)
            self.assertIsInstance(result, list, 'dut=%s' % dut)
            self.assertEqual(len(result), 1, 'dut=%s' % dut)
            self.assertEqual(result[0], {}, 'dut=%s' % dut)

            result = dut.run_commands('show running-config | include %s$' %
                                      hostname, 'text')
            self.assertEqual(result[0]['output'].strip(), hostname)

    def test_config_single_multiline_command(self):
        for dut in self.duts:
            # Clear any current banner
            dut.config('no banner login')

            banner = 'This is a new banner\nwith different lines!!!'
            cmd = 'banner login MULTILINE:%s' % banner
            result = dut.config(cmd)
            self.assertIsInstance(result, list, 'dut=%s' % dut)
            self.assertEqual(len(result), 1, 'dut=%s' % dut)
            self.assertEqual(result[0], {}, 'dut=%s' % dut)
            result = dut.run_commands('show banner login', 'text')
            self.assertEqual(result[0]['output'].strip().split('\n'),
                             banner.split('\n'))

    def test_config_multiple_commands(self):
        for dut in self.duts:
            commands = list()
            for i in range(1, random_int(10, 200)):
                commands.append('hostname %s' % random_string(5, 20))
            result = dut.config(commands[:])
            self.assertIsInstance(result, list, 'dut=%s' % dut)
            self.assertEqual(len(result), len(commands), 'dut=%s' % dut)

    def test_multiple_requests(self):
        for dut in self.duts:
            for i in range(1, random_int(10, 200)):
                result = dut.run_commands('show version')
                self.assertIsInstance(result, list, 'dut=%s' % dut)
                self.assertEqual(len(result), 1, 'dut=%s' % dut)

    def test_get_block(self):
        # Verify get_block using a config string returns correct value
        for dut in self.duts:
            api = dut.api('interfaces')
            config = api.config
            running = api.get_block('interface Ethernet1')
            txtstr = api.get_block('interface Ethernet1', config=config)
            self.assertEqual(running, txtstr)

    def test_get_block_none(self):
        # Verify get_block using a config string where match fails returns None
        for dut in self.duts:
            api = dut.api('interfaces')
            txtstr = api.get_block('interface Ethernet1', config='config')
            self.assertEqual(txtstr, None)

    def tearDown(self):
        for dut in self.duts:
            dut.config("no enable secret")

class TestNode(unittest.TestCase):

    def setUp(self):
        pyeapi.client.load_config(filename=get_fixture('dut.conf'))
        config = pyeapi.client.config

        self.duts = list()
        for name in config.sections():
            if name.startswith('connection:') and 'localhost' not in name:
                name = name.split(':')[1]
                self.duts.append(pyeapi.client.connect_to(name))

    def test_exception_trace(self):
        # Send commands that will return an error and validate the errors

        # General format of an error message:
        rfmt = r'Error \[%d\]: CLI command \d+ of \d+ \'.*\' failed: %s \[%s\]'
        # Design error tests
        cases = []
        # Send an incomplete command
        cases.append(('show run', rfmt
                      % (1002, 'invalid command',
                         'incomplete token \(at token \d+: \'.*\'\)')))
        # Send a mangled command
        cases.append(('shwo version', rfmt
                      % (1002, 'invalid command',
                         'Invalid input \(at token \d+: \'.*\'\)')))
        # Send a command that cannot be run through the api
        # note the command for reload looks to change in new EOS
        # in 4.15 the  reload now is replaced with 'force' if you are
        # testing some DUT running older code and this test fails
        # change the error message to the following:
        # To reload the machine over the API, please use 'reload now' instead
        cases.append(('reload', rfmt
                      % (1004, 'incompatible command',
                         'Command not permitted via API access..*')))
        # Send a continuous command that requires a break
        cases.append(('watch 10 show int e1 count rates', rfmt
                      % (1000, 'could not run command',
                         'init error.*')))
        # Send a command that has insufficient priv
        cases.append(('show running-config', rfmt
                      % (1002, 'invalid command',
                         'Invalid input \(privileged mode required\)')))

        for dut in self.duts:
            for (cmd, regex) in cases:
                try:
                    # Insert the error in list of valid commands
                    if cmd != "show running-config":
                        dut.enable(['show version', cmd, 'show hostname'],
                                   strict=True)
                    else:
                        dut.enable(['disable', 'show version', cmd],
                                   strict=True, send_enable=False)

                    self.fail('A CommandError should have been raised')
                except pyeapi.eapilib.CommandError as exc:
                    # Validate the properties of the exception
                    if cmd != 'show running-config':
                        self.assertEqual(len(exc.trace), 4)
                    else:
                        self.assertEqual(len(exc.trace), 3)
                    self.assertIsNotNone(exc.command_error)
                    self.assertIsNotNone(exc.output)
                    self.assertIsNotNone(exc.commands)
                    self.assertRegexpMatches(exc.message, regex)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Added version dependent system test for autocomplete feature.<commit_after>#
# Copyright (c) 2014, Arista Networks, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#   Redistributions of source code must retain the above copyright notice,
#   this list of conditions and the following disclaimer.
#
#   Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
#
#   Neither the name of Arista Networks nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL ARISTA NETWORKS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
import os
import unittest

import sys
sys.path.append(os.path.join(os.path.dirname(__file__), '../lib'))

from testlib import random_int, random_string, get_fixture

import pyeapi.client
import pyeapi.eapilib

class TestClient(unittest.TestCase):

    def setUp(self):
        pyeapi.client.load_config(filename=get_fixture('dut.conf'))
        config = pyeapi.client.config

        self.duts = list()
        for name in config.sections():
            if name.startswith('connection:') and 'localhost' not in name:
                name = name.split(':')[1]
                dut = pyeapi.client.connect_to(name)
                self.duts.append(dut)
                if dut._enablepwd is not None:
                    # If enable password defined for dut, set the
                    # enable password on the dut and clear it on tearDown
                    dut.config("enable secret %s" % dut._enablepwd)

    def test_enable_single_command(self):
        for dut in self.duts:
            result = dut.run_commands('show version')
            self.assertIsInstance(result, list, 'dut=%s' % dut)
            self.assertEqual(len(result), 1, 'dut=%s' % dut)

    def test_no_enable_single_command(self):
        for dut in self.duts:
            result = dut.run_commands('show version', 'json', send_enable=False)
            self.assertIsInstance(result, list, 'dut=%s' % dut)
            self.assertEqual(len(result), 1, 'dut=%s' % dut)

    def test_no_enable_single_command_no_auth(self):
        for dut in self.duts:
            with self.assertRaises(pyeapi.eapilib.CommandError):
                dut.run_commands(['disable', 'show running-config'], 'json', send_enable=False)

    def test_enable_multiple_commands(self):
        for dut in self.duts:
            commands = list()
            for i in range(1, random_int(10, 200)):
                commands.append('show version')
            result = dut.run_commands(commands[:])
            self.assertIsInstance(result, list, 'dut=%s' % dut)
            self.assertEqual(len(result), len(commands), 'dut=%s' % dut)

    def test_config_single_command(self):
        for dut in self.duts:
            hostname = 'hostname %s' % random_string(5, 50)
            result = dut.config(hostname)
            self.assertIsInstance(result, list, 'dut=%s' % dut)
            self.assertEqual(len(result), 1, 'dut=%s' % dut)
            self.assertEqual(result[0], {}, 'dut=%s' % dut)

            result = dut.run_commands('show running-config | include %s$' %
                                      hostname, 'text')
            self.assertEqual(result[0]['output'].strip(), hostname)

    def test_config_single_multiline_command(self):
        for dut in self.duts:
            # Clear any current banner
            dut.config('no banner login')

            banner = 'This is a new banner\nwith different lines!!!'
            cmd = 'banner login MULTILINE:%s' % banner
            result = dut.config(cmd)
            self.assertIsInstance(result, list, 'dut=%s' % dut)
            self.assertEqual(len(result), 1, 'dut=%s' % dut)
            self.assertEqual(result[0], {}, 'dut=%s' % dut)
            result = dut.run_commands('show banner login', 'text')
            self.assertEqual(result[0]['output'].strip().split('\n'),
                             banner.split('\n'))

    def test_config_multiple_commands(self):
        for dut in self.duts:
            commands = list()
            for i in range(1, random_int(10, 200)):
                commands.append('hostname %s' % random_string(5, 20))
            result = dut.config(commands[:])
            self.assertIsInstance(result, list, 'dut=%s' % dut)
            self.assertEqual(len(result), len(commands), 'dut=%s' % dut)

    def test_multiple_requests(self):
        for dut in self.duts:
            for i in range(1, random_int(10, 200)):
                result = dut.run_commands('show version')
                self.assertIsInstance(result, list, 'dut=%s' % dut)
                self.assertEqual(len(result), 1, 'dut=%s' % dut)

    def test_get_block(self):
        # Verify get_block using a config string returns correct value
        for dut in self.duts:
            api = dut.api('interfaces')
            config = api.config
            running = api.get_block('interface Ethernet1')
            txtstr = api.get_block('interface Ethernet1', config=config)
            self.assertEqual(running, txtstr)

    def test_get_block_none(self):
        # Verify get_block using a config string where match fails returns None
        for dut in self.duts:
            api = dut.api('interfaces')
            txtstr = api.get_block('interface Ethernet1', config='config')
            self.assertEqual(txtstr, None)

    def test_execute_with_autocomplete(self):
        # There are some versions of EOS before 4.17.x that have the
        # autocomplete feature available. If system tests are run on one of
        # those version of EOS this system test will fail.
        for dut in self.duts:
            result = dut.connection.execute(['show version'], encoding='json')
            version = result['result'][0]['version']
            version = version.split('.')
            if int(version[0]) >= 4 and int(version[1]) >= 17:
                result = dut.connection.execute(['sh ver'], encoding='json',
                                                autoComplete=True)
                self.assertIn('version', result['result'][0])
            else:
                # Verify exception thrown for EOS version that does not
                # support autocomplete parameter with EAPI
                with self.assertRaises(pyeapi.eapilib.CommandError):
                    dut.connection.execute(['sh ver'], encoding='json',
                                           autoComplete=True)

    def tearDown(self):
        for dut in self.duts:
            dut.config("no enable secret")

class TestNode(unittest.TestCase):

    def setUp(self):
        pyeapi.client.load_config(filename=get_fixture('dut.conf'))
        config = pyeapi.client.config

        self.duts = list()
        for name in config.sections():
            if name.startswith('connection:') and 'localhost' not in name:
                name = name.split(':')[1]
                self.duts.append(pyeapi.client.connect_to(name))

    def test_exception_trace(self):
        # Send commands that will return an error and validate the errors

        # General format of an error message:
        rfmt = r'Error \[%d\]: CLI command \d+ of \d+ \'.*\' failed: %s \[%s\]'
        # Design error tests
        cases = []
        # Send an incomplete command
        cases.append(('show run', rfmt
                      % (1002, 'invalid command',
                         'incomplete token \(at token \d+: \'.*\'\)')))
        # Send a mangled command
        cases.append(('shwo version', rfmt
                      % (1002, 'invalid command',
                         'Invalid input \(at token \d+: \'.*\'\)')))
        # Send a command that cannot be run through the api
        # note the command for reload looks to change in new EOS
        # in 4.15 the  reload now is replaced with 'force' if you are
        # testing some DUT running older code and this test fails
        # change the error message to the following:
        # To reload the machine over the API, please use 'reload now' instead
        cases.append(('reload', rfmt
                      % (1004, 'incompatible command',
                         'Command not permitted via API access..*')))
        # Send a continuous command that requires a break
        cases.append(('watch 10 show int e1 count rates', rfmt
                      % (1000, 'could not run command',
                         'init error.*')))
        # Send a command that has insufficient priv
        cases.append(('show running-config', rfmt
                      % (1002, 'invalid command',
                         'Invalid input \(privileged mode required\)')))

        for dut in self.duts:
            for (cmd, regex) in cases:
                try:
                    # Insert the error in list of valid commands
                    if cmd != "show running-config":
                        dut.enable(['show version', cmd, 'show hostname'],
                                   strict=True)
                    else:
                        dut.enable(['disable', 'show version', cmd],
                                   strict=True, send_enable=False)

                    self.fail('A CommandError should have been raised')
                except pyeapi.eapilib.CommandError as exc:
                    # Validate the properties of the exception
                    if cmd != 'show running-config':
                        self.assertEqual(len(exc.trace), 4)
                    else:
                        self.assertEqual(len(exc.trace), 3)
                    self.assertIsNotNone(exc.command_error)
                    self.assertIsNotNone(exc.output)
                    self.assertIsNotNone(exc.commands)
                    self.assertRegexpMatches(exc.message, regex)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import os
import sys
import json
import aiohttp
import asyncio

import checks
import load_config
from config.server_config import ServerConfig

import discord
from discord.ext.commands import bot
from discord.ext.commands import group

class Settings():
	"""
	Settings is a mix of settings and admin stuff for the bot.
	"""
	def __init__(self, Bot):
		self.bot = Bot
		self.con = ServerConfig()
		self.servers = self.con.servers

	@bot.command(pass_context=True, hidden=True)
	@checks.is_bot_owner()
	async def blacklist(self, ctx, option, *args):
		"""
		Usage:
			.blacklist [ + | - | add | remove ] @UserName [@UserName2 ...]
		Add or remove users to the blacklist.
		Blacklisted users are forbidden from using bot commands.
		Only the bot owner can use this command
		"""
		blacklist_amount = 0
		mentions = ctx.message.mentions

		if not mentions:
			return await self.bot.say("You didn't mention anyone")

		if option not in ['+', '-', 'add', 'remove']:
			return await self.bot.say('Invalid option "%s" specified, use +, -, add, or remove' % option, expire_in=20)

		for user in mentions:
			if user.id == load_config.owner:
				print("[Commands:Blacklist] The owner cannot be blacklisted.")
				await self.bot.say("The owner cannot be blacklisted.")
				mentions.remove(user)

		if option in ['+', 'add']:
			with open("config/blacklist.txt", "r") as fp:
				for user in mentions:
					for line in fp.readlines():
						if user.id + "\n" in line:
							mentions.remove(user)

			with open("config/blacklist.txt", "a+") as fp:
				lines = fp.readlines()
				for user in mentions:
					if user.id not in lines:
						fp.write("{}\n".format(user.id))
						blacklist_amount += 1
			return await self.bot.say('{} user(s) have been added to the blacklist'.format(blacklist_amount))

		elif option in ['-', 'remove']:
			with open("config/blacklist.txt", "r") as fp:
				lines = fp.readlines()
			with open("config/blacklist.txt", "w") as fp:
				for user in mentions:
					for line in lines:
						if user.id + "\n" != line:
							fp.write(line)
						else:
							fp.write("")
							blacklist_amount += 1
				return await self.bot.say('{} user(s) have been removed from the blacklist'.format(blacklist_amount))

	@bot.command(pass_context=True, hidden=True)
	@checks.is_bot_owner()
	async def enablesetting(self, ctx, setting):
		server_id = ctx.message.server.id
		if setting in self.serverconfig[server_id]:
			self.serverconfig = self.con.load_config()
			if not self.serverconfig[server_id][setting]["enabled"]:
				self.serverconfig[server_id][setting]["enabled"] = 1
				self.con.update_config(self.serverconfig)
				return await self.bot.say("'{}' was enabled!".format(setting))
			else:
				self.serverconfig[server_id][setting]["enabled"] = 0
				self.con.update_config(self.serverconfig)
				return await self.bot.say("'{}' was disabled :cry:".format(setting))
		else:
			return await self.bot.say("That module dont exist fam. You made the thing")

	@bot.command(pass_context=True)
	@checks.is_bot_owner()
	async def printsettings(self, ctx, setting=None):
		self.serverconfig = self.con.load_config()
		config = self.serverconfig[ctx.message.server.id]
		em = discord.Embed(colour=0xDEADBF)
		em.set_author(name="RoxBot settings for {}.".format(ctx.message.server.name), icon_url=self.bot.user.avatar_url)

		for settings in config:
			settingcontent = ""
			for x in config[settings].items():
				settingcontent += str(x).strip("()") + "\n"
			em.add_field(name=settings, value=settingcontent)

		return await self.bot.say(embed=em)

	@group(pass_context=True, hidden=True)
	@checks.is_bot_owner()
	async def set(self, ctx):
		if ctx.invoked_subcommand is None:
			return await self.bot.say('Missing Argument')

	@set.command(pass_context=True, hidden=True)
	async def welcomechannel(self, ctx, channel: discord.Channel = None):
		self.serverconfig = self.con.load_config()
		self.serverconfig[ctx.message.server.id]["greets"]["welcome-channel"] = channel.id
		self.con.update_config(self.serverconfig)
		return await self.bot.say("{} has been set as the welcome channel!".format(channel.mention))

	@set.command(pass_context=True, hidden=True)
	async def goodbyechannel(self, ctx, channel: discord.Channel = None):
		self.serverconfig = self.con.load_config()
		self.serverconfig[ctx.message.server.id]["goodbyes"]["goodbye-channel"] = channel.id
		self.con.update_config(self.serverconfig)
		return await self.bot.say("{} has been set as the goodbye channel!".format(channel.mention))

	@set.command(pass_context=True, hidden=True)
	async def twitchchannel(self, ctx, channel: discord.Channel = None):
		self.serverconfig = self.con.load_config()
		self.serverconfig[ctx.message.server.id]["twitch"]["twitch-channel"] = channel.id
		self.con.update_config(self.serverconfig)
		return await self.bot.say("{} has been set as the twitch shilling channel!".format(channel.mention))

	@set.command(pass_context=True, hidden=True)
	async def welcomemessage(self, ctx, *, message: str):
		self.serverconfig = self.con.load_config()
		self.serverconfig[ctx.message.server.id]["greets"]["custom-message"] = message
		self.con.update_config(self.serverconfig)
		return await self.bot.say("Custom message set to '{}'".format(message))

	@set.command(pass_context=True, hidden=True)
	async def muterole(self, ctx, role: discord.Role = None):
		self.serverconfig = self.con.load_config()
		self.serverconfig[ctx.message.server.id]["mute"]["role"] = role.id
		self.con.update_config(self.serverconfig)
		return await self.bot.say("Muted role set to '{}'".format(role.name))

	@set.command(pass_context=True, hidden=True)
	async def muteadmin(self, ctx, role: discord.Role = None):
		self.serverconfig = self.con.load_config()
		self.serverconfig[ctx.message.server.id]["mute"]["admin-role"].append(role.id)
		self.con.update_config(self.serverconfig)
		return await self.bot.say("Admin role appended to list: '{}'".format(role.name))

	@bot.command(pass_context=True, hidden=True, aliases=["setava"])
	@checks.is_bot_owner()
	async def changeavatar(self, ctx, url=None):
		"""
		Usage:
			{command_prefix}setavatar [url]
		Changes the bot's avatar.
		Attaching a file and leaving the url parameter blank also works.
		"""
		if ctx.message.attachments:
			thing = ctx.message.attachments[0]['url']
		else:
			thing = url.strip('<>')

		avaimg = 'avaimg.png'
		async with aiohttp.ClientSession() as session:
			with session.get(thing) as img:
				with open(avaimg, 'wb') as f:
					f.write(await img.read())
		with open(avaimg, 'rb') as f:
			await self.bot.edit_profile(avatar=f.read())
		os.remove(avaimg)
		asyncio.sleep(2)
		return await self.bot.say(":ok_hand:")

	@bot.command(pass_context=True, hidden=True, aliases=["nick"])
	@checks.is_bot_owner()
	async def changenickname(self, ctx, *nick):
		if ctx.message.channel.permissions_for(ctx.message.server.me).change_nickname:
			await self.bot.change_nickname(ctx.message.server.me, ' '.join(nick))
			return await self.bot.say(":thumbsup:")
		else:
			return await self.bot.say("I don't have permission to do that :sob:", delete_after=self.con.delete_after)

	@bot.command(pass_context=True, hidden=True, aliases=["setgame", "game"])
	@checks.is_bot_owner()
	async def changegame(self, ctx, *, game: str):
		if game.lower() == "none":
			game_name = None
		else:
			game_name = discord.Game(name=game)
		await self.bot.change_presence(game=game_name, afk=False)
		return await self.bot.say(":ok_hand: Game set to {}".format(str(game_name)))

	@bot.command(pass_context=True, hidden=True, aliases=["status"])
	@checks.is_bot_owner()
	async def changestatus(self, ctx, status: str):
		status = status.lower()
		if status == 'offline' or status == 'invisible':
			discordStatus = discord.Status.invisible
		elif status == 'idle':
			discordStatus = discord.Status.idle
		elif status == 'dnd':
			discordStatus = discord.Status.dnd
		else:
			discordStatus = discord.Status.online
		await self.bot.change_presence(status=discordStatus)
		await self.bot.say("**:ok:** Status set to {}".format(discordStatus))

	@bot.command(pass_context=True, hidden=True)
	@checks.is_bot_owner()
	async def echo(self, ctx, channel, *, message: str):
		if ctx.message.channel_mentions:
			for channel in ctx.message.channel_mentions:
				await self.bot.send_message(channel, content=message)
			return await self.bot.say(":point_left:")
		elif channel.isdigit():
			channel = ctx.message.server.get_channel(channel)
			await self.bot.send_message(channel, content=message)
			return await self.bot.say(":point_left:")
		else:
			return await self.bot.say("You did something wrong smh")

	@bot.command(pass_context=True, hidden=True)
	@checks.is_bot_owner()
	async def restart(self, ctx):
		await self.bot.logout()
		return os.execl(sys.executable, sys.executable, *sys.argv)

	@bot.command(pass_context=True, hidden=True)
	@checks.is_bot_owner()
	async def shutdown(self, ctx):
		await self.bot.logout()
		return exit(0)

	@bot.command(pass_context=True, hidden=True)
	@checks.is_bot_owner()
	async def announce(self, ctx, *announcement):
		"""
		ONLY USE FOR SERIOUS ANNOUNCEMENTS
		"""
		# TODO: Make colour top level role colour
		# TODO: Custom message for annoucement footer
		embed = discord.Embed(title="RoxBot Announcement", colour=discord.Colour(0x306f99), description=' '.join(announcement))
		embed.set_footer(text="This message has be automatically generated by a cute ass Roxie",
						 icon_url=self.bot.user.avatar_url)
		for server in self.bot.servers:
			await self.bot.send_message(server, embed=embed)
		return await self.bot.say("Done!", delete_after=self.con.delete_after)

def setup(Bot):
	Bot.add_cog(Settings(Bot))<commit_msg>print settings output is nicer now woo<commit_after>import os
import sys
import json
import aiohttp
import asyncio

import checks
import load_config
from config.server_config import ServerConfig

import discord
from discord.ext.commands import bot
from discord.ext.commands import group

class Settings():
	"""
	Settings is a mix of settings and admin stuff for the bot.
	"""
	def __init__(self, Bot):
		self.bot = Bot
		self.con = ServerConfig()
		self.servers = self.con.servers

	@bot.command(pass_context=True, hidden=True)
	@checks.is_bot_owner()
	async def blacklist(self, ctx, option, *args):
		"""
		Usage:
			.blacklist [ + | - | add | remove ] @UserName [@UserName2 ...]
		Add or remove users to the blacklist.
		Blacklisted users are forbidden from using bot commands.
		Only the bot owner can use this command
		"""
		blacklist_amount = 0
		mentions = ctx.message.mentions

		if not mentions:
			return await self.bot.say("You didn't mention anyone")

		if option not in ['+', '-', 'add', 'remove']:
			return await self.bot.say('Invalid option "%s" specified, use +, -, add, or remove' % option, expire_in=20)

		for user in mentions:
			if user.id == load_config.owner:
				print("[Commands:Blacklist] The owner cannot be blacklisted.")
				await self.bot.say("The owner cannot be blacklisted.")
				mentions.remove(user)

		if option in ['+', 'add']:
			with open("config/blacklist.txt", "r") as fp:
				for user in mentions:
					for line in fp.readlines():
						if user.id + "\n" in line:
							mentions.remove(user)

			with open("config/blacklist.txt", "a+") as fp:
				lines = fp.readlines()
				for user in mentions:
					if user.id not in lines:
						fp.write("{}\n".format(user.id))
						blacklist_amount += 1
			return await self.bot.say('{} user(s) have been added to the blacklist'.format(blacklist_amount))

		elif option in ['-', 'remove']:
			with open("config/blacklist.txt", "r") as fp:
				lines = fp.readlines()
			with open("config/blacklist.txt", "w") as fp:
				for user in mentions:
					for line in lines:
						if user.id + "\n" != line:
							fp.write(line)
						else:
							fp.write("")
							blacklist_amount += 1
				return await self.bot.say('{} user(s) have been removed from the blacklist'.format(blacklist_amount))

	@bot.command(pass_context=True, hidden=True)
	@checks.is_bot_owner()
	async def enablesetting(self, ctx, setting):
		server_id = ctx.message.server.id
		if setting in self.serverconfig[server_id]:
			self.serverconfig = self.con.load_config()
			if not self.serverconfig[server_id][setting]["enabled"]:
				self.serverconfig[server_id][setting]["enabled"] = 1
				self.con.update_config(self.serverconfig)
				return await self.bot.say("'{}' was enabled!".format(setting))
			else:
				self.serverconfig[server_id][setting]["enabled"] = 0
				self.con.update_config(self.serverconfig)
				return await self.bot.say("'{}' was disabled :cry:".format(setting))
		else:
			return await self.bot.say("That module dont exist fam. You made the thing")

	@bot.command(pass_context=True)
	@checks.is_bot_owner()
	async def printsettings(self, ctx):
		self.serverconfig = self.con.load_config()
		config = self.serverconfig[ctx.message.server.id]
		em = discord.Embed(colour=0xDEADBF)
		em.set_author(name="RoxBot settings for {}.".format(ctx.message.server.name), icon_url=self.bot.user.avatar_url)

		for settings in config:
			settingcontent = ""
			for x in config[settings].items():
				settingcontent += str(x).strip("()") + "\n"
			em.add_field(name=settings, value=settingcontent)

		return await self.bot.say(embed=em)

	@group(pass_context=True, hidden=True)
	@checks.is_bot_owner()
	async def set(self, ctx):
		if ctx.invoked_subcommand is None:
			return await self.bot.say('Missing Argument')

	@set.command(pass_context=True, hidden=True)
	async def welcomechannel(self, ctx, channel: discord.Channel = None):
		self.serverconfig = self.con.load_config()
		self.serverconfig[ctx.message.server.id]["greets"]["welcome-channel"] = channel.id
		self.con.update_config(self.serverconfig)
		return await self.bot.say("{} has been set as the welcome channel!".format(channel.mention))

	@set.command(pass_context=True, hidden=True)
	async def goodbyechannel(self, ctx, channel: discord.Channel = None):
		self.serverconfig = self.con.load_config()
		self.serverconfig[ctx.message.server.id]["goodbyes"]["goodbye-channel"] = channel.id
		self.con.update_config(self.serverconfig)
		return await self.bot.say("{} has been set as the goodbye channel!".format(channel.mention))

	@set.command(pass_context=True, hidden=True)
	async def twitchchannel(self, ctx, channel: discord.Channel = None):
		self.serverconfig = self.con.load_config()
		self.serverconfig[ctx.message.server.id]["twitch"]["twitch-channel"] = channel.id
		self.con.update_config(self.serverconfig)
		return await self.bot.say("{} has been set as the twitch shilling channel!".format(channel.mention))

	@set.command(pass_context=True, hidden=True)
	async def welcomemessage(self, ctx, *, message: str):
		self.serverconfig = self.con.load_config()
		self.serverconfig[ctx.message.server.id]["greets"]["custom-message"] = message
		self.con.update_config(self.serverconfig)
		return await self.bot.say("Custom message set to '{}'".format(message))

	@set.command(pass_context=True, hidden=True)
	async def muterole(self, ctx, role: discord.Role = None):
		self.serverconfig = self.con.load_config()
		self.serverconfig[ctx.message.server.id]["mute"]["role"] = role.id
		self.con.update_config(self.serverconfig)
		return await self.bot.say("Muted role set to '{}'".format(role.name))

	@set.command(pass_context=True, hidden=True)
	async def muteadmin(self, ctx, role: discord.Role = None):
		self.serverconfig = self.con.load_config()
		self.serverconfig[ctx.message.server.id]["mute"]["admin-role"].append(role.id)
		self.con.update_config(self.serverconfig)
		return await self.bot.say("Admin role appended to list: '{}'".format(role.name))

	@bot.command(pass_context=True, hidden=True, aliases=["setava"])
	@checks.is_bot_owner()
	async def changeavatar(self, ctx, url=None):
		"""
		Usage:
			{command_prefix}setavatar [url]
		Changes the bot's avatar.
		Attaching a file and leaving the url parameter blank also works.
		"""
		if ctx.message.attachments:
			thing = ctx.message.attachments[0]['url']
		else:
			thing = url.strip('<>')

		avaimg = 'avaimg.png'
		async with aiohttp.ClientSession() as session:
			with session.get(thing) as img:
				with open(avaimg, 'wb') as f:
					f.write(await img.read())
		with open(avaimg, 'rb') as f:
			await self.bot.edit_profile(avatar=f.read())
		os.remove(avaimg)
		asyncio.sleep(2)
		return await self.bot.say(":ok_hand:")

	@bot.command(pass_context=True, hidden=True, aliases=["nick"])
	@checks.is_bot_owner()
	async def changenickname(self, ctx, *nick):
		if ctx.message.channel.permissions_for(ctx.message.server.me).change_nickname:
			await self.bot.change_nickname(ctx.message.server.me, ' '.join(nick))
			return await self.bot.say(":thumbsup:")
		else:
			return await self.bot.say("I don't have permission to do that :sob:", delete_after=self.con.delete_after)

	@bot.command(pass_context=True, hidden=True, aliases=["setgame", "game"])
	@checks.is_bot_owner()
	async def changegame(self, ctx, *, game: str):
		if game.lower() == "none":
			game_name = None
		else:
			game_name = discord.Game(name=game)
		await self.bot.change_presence(game=game_name, afk=False)
		return await self.bot.say(":ok_hand: Game set to {}".format(str(game_name)))

	@bot.command(pass_context=True, hidden=True, aliases=["status"])
	@checks.is_bot_owner()
	async def changestatus(self, ctx, status: str):
		status = status.lower()
		if status == 'offline' or status == 'invisible':
			discordStatus = discord.Status.invisible
		elif status == 'idle':
			discordStatus = discord.Status.idle
		elif status == 'dnd':
			discordStatus = discord.Status.dnd
		else:
			discordStatus = discord.Status.online
		await self.bot.change_presence(status=discordStatus)
		await self.bot.say("**:ok:** Status set to {}".format(discordStatus))

	@bot.command(pass_context=True, hidden=True)
	@checks.is_bot_owner()
	async def echo(self, ctx, channel, *, message: str):
		if ctx.message.channel_mentions:
			for channel in ctx.message.channel_mentions:
				await self.bot.send_message(channel, content=message)
			return await self.bot.say(":point_left:")
		elif channel.isdigit():
			channel = ctx.message.server.get_channel(channel)
			await self.bot.send_message(channel, content=message)
			return await self.bot.say(":point_left:")
		else:
			return await self.bot.say("You did something wrong smh")

	@bot.command(pass_context=True, hidden=True)
	@checks.is_bot_owner()
	async def restart(self, ctx):
		await self.bot.logout()
		return os.execl(sys.executable, sys.executable, *sys.argv)

	@bot.command(pass_context=True, hidden=True)
	@checks.is_bot_owner()
	async def shutdown(self, ctx):
		await self.bot.logout()
		return exit(0)

	@bot.command(pass_context=True, hidden=True)
	@checks.is_bot_owner()
	async def announce(self, ctx, *announcement):
		"""
		ONLY USE FOR SERIOUS ANNOUNCEMENTS
		"""
		# TODO: Make colour top level role colour
		# TODO: Custom message for annoucement footer
		embed = discord.Embed(title="RoxBot Announcement", colour=discord.Colour(0x306f99), description=' '.join(announcement))
		embed.set_footer(text="This message has be automatically generated by a cute ass Roxie",
						 icon_url=self.bot.user.avatar_url)
		for server in self.bot.servers:
			await self.bot.send_message(server, embed=embed)
		return await self.bot.say("Done!", delete_after=self.con.delete_after)

def setup(Bot):
	Bot.add_cog(Settings(Bot))<|endoftext|>


<commit_before>import collections, json, os, re
from bs4 import BeautifulSoup
from jinja2 import Environment, PackageLoader, FileSystemLoader

class HtmlGenerator(object):

    def __init__(self, platform, cordova, root, proj_home, src_dir, config, settings_config):
        self.platform = platform
        self.cordova = cordova
        self.root = root
        self.proj_home = proj_home
        self.src_dir = src_dir
        self.export_path = os.sep.join((src_dir, 'www'))
        self.plugin_templates = self._get_plugins_templates()
        self.config = config
        self.settings_config = settings_config

    def generate(self):
        '''
        generate the templates
        '''
        #the jinja templates of core, the jinja project templates, the jinja templates of plugins
        templates_path = {
            "core": os.sep.join((self.src_dir, 'templates')), "project":
            os.path.join(self.proj_home, 'src', 'templates'),
            "plugins": self.plugin_templates
        }
        #generate header footer data firstly
        header_data, footer_data = self._get_header_footer_data(templates_path)
        #generate the rest
        self._create_html(templates_path["core"], templates_path, header_data, footer_data)
        self._create_html(templates_path["project"], templates_path, header_data, footer_data)
        self._generate_settings(templates_path["core"], templates_path, header_data, footer_data)

        for d in self.plugin_templates:
            self._create_html(d, templates_path, header_data, footer_data)

    def _check_for_data(self, paths, filename):
        '''
        check if file exists in any of the paths
        '''
        _in_plugins = []
        for d in paths["plugins"]:
            if os.path.exists(os.path.join(d, filename)):
                _in_plugins.append(d)
        return _in_plugins

    def _check_for_template(self, name):
        '''
        check if template exists inside the paths
        '''
        res = []
        for t in self.plugin_templates:
            if name in os.listdir(t):
                res.append(t)
        return res

    def _create_html(self, current_path, paths, header_data, footer_data):
        environ = Environment(loader=FileSystemLoader(current_path))
        environ_core = Environment(loader=FileSystemLoader(paths["core"]))
        environ_project = Environment(loader=FileSystemLoader(paths["project"]))
        environ.globals["_get_letter"] = self._get_letter
        environ_project.globals["_get_letter"] = self._get_letter

        header_template = environ_core.get_template("header.html")
        footer_template = environ_core.get_template("footer.html")

        for path, dirs, files in os.walk(current_path):
            for f in files:
                if self._is_valid_file(f):
                    htmlfile = '{0}.html'.format(f.split(".")[0])
                    htmlfilepath = os.path.join(path, htmlfile)
                    jsonfilepath = os.path.join(path, f)

                    data = None
                    if current_path == paths["core"]:
                        #check if the same json exists in any of the plugins and merge it
                        data_in_plugins = self._check_for_data(paths, f)
                        if len(data_in_plugins)>0:
                            for p in data_in_plugins:
                                if data == None:
                                    data = self._get_data(current_path, f, p)
                                else:
                                    self._merge(data, self._get_data(current_path, f, p))

                    #merge with the data in json
                    if data:
                        self._do_merge(f, data, paths["project"])
                    else:
                        data = self._get_data(current_path, f, paths["project"])

                    #generate templates:
                    if "templates" in data:
                        self._generate_templates(environ, data["templates"])

                    if os.path.exists(htmlfilepath):
                        print "generating file {0}".format(htmlfile)

                        if "header" in data:
                            self._merge(data["header"], header_data)
                        else:
                            data["header"] = header_data

                        if "footer" in data:
                            if not _is_empty(data["footer"]):
                                footer_data2 = deepcopy(footer_data)
                                data["footer"] = self._merge(footer_data2, data["footer"])
                        else:
                            data["footer"] = footer_data

                        if "body" in data:
                            body = self._sorted(data["body"])
                        else:
                            body=""

                        template = environ.get_template(htmlfile)
                        indexheader_data = {"cordova": self.cordova, "title": header_data["title"]}

                        popups=[]
                        if "popups" in data:
                            for popup in data["popups"]:
                                res = self._check_for_template(data["popups"][popup]["template"])
                                if len(res) == 1:
                                    environ_popup = Environment(loader=FileSystemLoader(res[0]))
                                    popup_template = environ_popup.get_template(data["popups"][popup]["template"])
                                    print "POPUP: adding {0} popup from plugins in {1}".format(data["popups"][popup]["template"], htmlfile)
                                elif len(res) > 1:
                                    print "There popup template {0} exists more than once. This needs to be fixed.".format(data["popups"][popup]["template"])
                                    sys.exit()
                                else:
                                    popup_template = environ.get_template(data["popups"][popup]["template"])
                                    print "POPUP: adding {0} popup from core in {1}".format(data["popups"][popup]["template"], htmlfile)
                                popups.append(popup_template.render(data=data["popups"][popup]["data"]))

                        output = template.render(
                            header_data=indexheader_data,
                            body=body,
                            popups="\n".join(popups),
                            platform=self.platform,
                            header=header_template.render(
                                data=data["header"],
                                platform=self.platform),
                            footer=footer_template.render(
                                    data=data["footer"],
                                    platform=self.platform))
                        self._write_data(os.sep.join((self.export_path, htmlfile)), self._prettify(output, 2))

    def _do_merge(self, filename, data, path):
        '''
        function for merging data tha exist in different places
        e.g.
        '''
        if os.path.exists(path) and filename in os.listdir(path):
            with open(os.path.join(path, filename), 'r') as f:
                new_data = json.load(f, object_pairs_hook=collections.OrderedDict)
            print "DATA: merging {0}".format(os.path.join(path, filename))
            return self._merge(data, new_data)
        else:
            return data

    def _find_template(self, name):
        '''
        find template
        '''
        pths = {}
        for t in self._check_for_template(name):
            paths = t.split("/")
            pths[paths[len(paths)-3]]=t
        return pths

    def _generate_settings(self, current_path, paths, header_data, footer_data):
        '''
        generate setttings page
        '''
        environ = Environment(loader=FileSystemLoader(current_path))
        settings=[]
        #get all the settings templates from plugins
        settings_in_plugins = self._find_template('settings.html')
        if os.path.exists(os.path.join(paths["project"], 'settings.html')):
            settings_in_plugins["project"] = paths["project"]

        for plg in settings_in_plugins:
            settings_path = settings_in_plugins[plg]
            environ_settings = Environment(loader=FileSystemLoader(settings_path))
            tmpl = environ_settings.get_template('settings.html')
            data = {}
            if self.settings_config is not None:
                #this is needed for when the plugins come through bower
                if "fieldtrip-" in plg:
                    plg = plg.replace("fieldtrip-", "")
                if plg in self.settings_config.keys():
                    value = self.settings_config[plg]
                    if value.startswith('{'):
                        data = json.loads(value, object_pairs_hook=collections.OrderedDict)
                    else:
                        data = value
            settings.append(tmpl.render(settings=data))

        header_template = environ.get_template("header.html")
        footer_template = environ.get_template("footer.html")
        template  = environ.get_template('settings.html')
        output = template.render(settings="\n".join(settings),
                            config = self.config,
                            header=header_template.render(
                                data=header_data,
                                platform=self.platform),
                            footer=footer_template.render(
                                    data=footer_data,
                                    platform=self.platform))
        self._write_data(os.sep.join((self.export_path, 'settings.html')), self._prettify(output, 2))

    def _generate_templates(self, environ, templates):
        for templ in templates:
            print "TEMPLATE: generating template {0}".format(templates[templ])
            script_template = environ.get_template(templates[templ])
            self._write_data(os.path.join(self.export_path, 'templates', templates[templ]), script_template.render())

    def _get_data(self, path1, filename, path2):
        '''
        get data from different two paths and merge them
        '''
        with open(os.path.join(path1, filename),'r') as f:
            try:
                json_object = json.load(f, object_pairs_hook=collections.OrderedDict)
                return self._do_merge(filename, json_object, path2)
            except ValueError, e:
                print "There was problem with the json file {0}".format(os.path.join(path1, filename))
                sys.exit()

    def _get_letter(self, obj):
        '''
        Get the letter that corresponds to column in a jqm grid view based on the
        number of elements in obj: see http://api.jquerymobile.com/1.3/grid-layout/
        '''

        letter = 'a'
        if len(obj) > 1:
            i = len(obj) - 2
            letter = chr(i + ord('a'))

        return letter

    def _get_header_footer_data(self, templates_path):
        '''
        get header and footer data
        '''
        header_data = self._get_data(templates_path["core"], 'header.json', templates_path["project"])
        footer_data = self._get_data(templates_path["core"], 'footer.json', templates_path["project"])
        for d in templates_path["plugins"]:
            if os.path.exists(os.path.join(d, 'header.json')):
                self._do_merge('header.json', header_data, d)
            if os.path.exists(os.path.join(d, 'footer.json')):
                self._do_merge('footer.json', footer_data, d)
        return header_data, footer_data

    def _get_plugins_templates(self):
        '''
        get a list of directories with templates
        '''
        plugins_list = []
        #check the folders inside the plugins folder and check for templates folders
        for d in os.listdir(os.path.join(self.root, 'plugins')):
            d1 = os.path.join(self.root, 'plugins', d)
            for dire in os.listdir(d1):
                p = os.path.join(d1, dire)
                if os.path.isdir(p) and not dire.startswith("."):
                    tmpl_path = os.path.join(p, "templates")
                    if os.path.exists(tmpl_path):
                        plugins_list.append(tmpl_path)

        #check for plugins inside the project.json of the project folder in case there are bower plugins
        with open(os.path.join(self.src_dir, 'www', 'theme', 'project.json'),'r') as f:
            plgins = json.load(f)["plugins"]
            for k, v in plgins["fieldtrip"].iteritems():
                if re.match('^v?\d+.\d+.\d+', v):
                    tmpl_path = os.path.join('bower_components', 'fieldtrip-{0}'.format(k), 'src', 'templates')
                    if os.path.exists(tmpl_path):
                        plugins_list.append(os.path.join('bower_components', 'fieldtrip-{0}'.format(k), 'src', 'templates'))
        return plugins_list

    def _is_empty(self, any_structure):
        # TODO
        if any_structure:
            return False
        else:
            return True

    def _is_valid_file(self, f):
        return f.endswith("json") and not f in ["header.json", "footer.json", "settings.json"]

    def _merge(self, a, b, path=None):
        '''
        merges b into a
        #http://stackoverflow.com/questions/7204805/python-dictionaries-of-dictionaries-merge
        '''
        if path is None: path = []
        for key in b:
            if key in a:
                if isinstance(a[key], dict) and isinstance(b[key], dict):
                    if self._is_empty(b[key]):
                        a[key] = b[key]
                    else:
                        self._merge(a[key], b[key], path + [str(key)])
                elif a[key] == b[key]:
                    pass # same leaf value
                else:
                    a[key] = b[key]
            else:
                a[key] = b[key]
        return a

    def _prettify(self, output, indent='2'):
        '''
        custom indentation for BeautifulSoup
        '''
        soup = BeautifulSoup(output, "html5lib")
        if len(soup.findAll('script')) > 0:
            s = soup.prettify()
        else:
            s = soup.div.prettify()
        r = re.compile(r'^(\s*)', re.MULTILINE)
        return r.sub(r'\1\1', s)

    def _sorted(self, dic):
        '''
        sort letters
        dic --> dictionary
        '''
        dic = collections.OrderedDict(sorted(dic.items(), key=lambda t: t[0]))
        return dic

    def _write_data(self, fil, filedata):
        '''
        fil --> filename
        filedata --> content that will be written in filename
        '''
        f = open(fil, 'w')
        f.write(filedata.encode('utf-8'))
        f.close()<commit_msg>Add missing imports sys, deepcopy<commit_after>import collections
import json
import os
import re
import sys
from copy import deepcopy
from bs4 import BeautifulSoup
from jinja2 import Environment, FileSystemLoader

class HtmlGenerator(object):

    def __init__(self, platform, cordova, root, proj_home, src_dir, config, settings_config):
        self.platform = platform
        self.cordova = cordova
        self.root = root
        self.proj_home = proj_home
        self.src_dir = src_dir
        self.export_path = os.sep.join((src_dir, 'www'))
        self.plugin_templates = self._get_plugins_templates()
        self.config = config
        self.settings_config = settings_config

    def generate(self):
        '''
        generate the templates
        '''
        #the jinja templates of core, the jinja project templates, the jinja templates of plugins
        templates_path = {
            "core": os.sep.join((self.src_dir, 'templates')), "project":
            os.path.join(self.proj_home, 'src', 'templates'),
            "plugins": self.plugin_templates
        }
        #generate header footer data firstly
        header_data, footer_data = self._get_header_footer_data(templates_path)
        #generate the rest
        self._create_html(templates_path["core"], templates_path, header_data, footer_data)
        self._create_html(templates_path["project"], templates_path, header_data, footer_data)
        self._generate_settings(templates_path["core"], templates_path, header_data, footer_data)

        for d in self.plugin_templates:
            self._create_html(d, templates_path, header_data, footer_data)

    def _check_for_data(self, paths, filename):
        '''
        check if file exists in any of the paths
        '''
        _in_plugins = []
        for d in paths["plugins"]:
            if os.path.exists(os.path.join(d, filename)):
                _in_plugins.append(d)
        return _in_plugins

    def _check_for_template(self, name):
        '''
        check if template exists inside the paths
        '''
        res = []
        for t in self.plugin_templates:
            if name in os.listdir(t):
                res.append(t)
        return res

    def _create_html(self, current_path, paths, header_data, footer_data):
        environ = Environment(loader=FileSystemLoader(current_path))
        environ_core = Environment(loader=FileSystemLoader(paths["core"]))
        environ_project = Environment(loader=FileSystemLoader(paths["project"]))
        environ.globals["_get_letter"] = self._get_letter
        environ_project.globals["_get_letter"] = self._get_letter

        header_template = environ_core.get_template("header.html")
        footer_template = environ_core.get_template("footer.html")

        for path, dirs, files in os.walk(current_path):
            for f in files:
                if self._is_valid_file(f):
                    htmlfile = '{0}.html'.format(f.split(".")[0])
                    htmlfilepath = os.path.join(path, htmlfile)
                    jsonfilepath = os.path.join(path, f)

                    data = None
                    if current_path == paths["core"]:
                        #check if the same json exists in any of the plugins and merge it
                        data_in_plugins = self._check_for_data(paths, f)
                        if len(data_in_plugins)>0:
                            for p in data_in_plugins:
                                if data == None:
                                    data = self._get_data(current_path, f, p)
                                else:
                                    self._merge(data, self._get_data(current_path, f, p))

                    #merge with the data in json
                    if data:
                        self._do_merge(f, data, paths["project"])
                    else:
                        data = self._get_data(current_path, f, paths["project"])

                    #generate templates:
                    if "templates" in data:
                        self._generate_templates(environ, data["templates"])

                    if os.path.exists(htmlfilepath):
                        print "generating file {0}".format(htmlfile)

                        if "header" in data:
                            self._merge(data["header"], header_data)
                        else:
                            data["header"] = header_data

                        if "footer" in data:
                            if not _is_empty(data["footer"]):
                                footer_data2 = deepcopy(footer_data)
                                data["footer"] = self._merge(footer_data2, data["footer"])
                        else:
                            data["footer"] = footer_data

                        if "body" in data:
                            body = self._sorted(data["body"])
                        else:
                            body=""

                        template = environ.get_template(htmlfile)
                        indexheader_data = {"cordova": self.cordova, "title": header_data["title"]}

                        popups=[]
                        if "popups" in data:
                            for popup in data["popups"]:
                                res = self._check_for_template(data["popups"][popup]["template"])
                                if len(res) == 1:
                                    environ_popup = Environment(loader=FileSystemLoader(res[0]))
                                    popup_template = environ_popup.get_template(data["popups"][popup]["template"])
                                    print "POPUP: adding {0} popup from plugins in {1}".format(data["popups"][popup]["template"], htmlfile)
                                elif len(res) > 1:
                                    print "There popup template {0} exists more than once. This needs to be fixed.".format(data["popups"][popup]["template"])
                                    sys.exit()
                                else:
                                    popup_template = environ.get_template(data["popups"][popup]["template"])
                                    print "POPUP: adding {0} popup from core in {1}".format(data["popups"][popup]["template"], htmlfile)
                                popups.append(popup_template.render(data=data["popups"][popup]["data"]))

                        output = template.render(
                            header_data=indexheader_data,
                            body=body,
                            popups="\n".join(popups),
                            platform=self.platform,
                            header=header_template.render(
                                data=data["header"],
                                platform=self.platform),
                            footer=footer_template.render(
                                    data=data["footer"],
                                    platform=self.platform))
                        self._write_data(os.sep.join((self.export_path, htmlfile)), self._prettify(output, 2))

    def _do_merge(self, filename, data, path):
        '''
        function for merging data tha exist in different places
        e.g.
        '''
        if os.path.exists(path) and filename in os.listdir(path):
            with open(os.path.join(path, filename), 'r') as f:
                new_data = json.load(f, object_pairs_hook=collections.OrderedDict)
            print "DATA: merging {0}".format(os.path.join(path, filename))
            return self._merge(data, new_data)
        else:
            return data

    def _find_template(self, name):
        '''
        find template
        '''
        pths = {}
        for t in self._check_for_template(name):
            paths = t.split("/")
            pths[paths[len(paths)-3]]=t
        return pths

    def _generate_settings(self, current_path, paths, header_data, footer_data):
        '''
        generate setttings page
        '''
        environ = Environment(loader=FileSystemLoader(current_path))
        settings=[]
        #get all the settings templates from plugins
        settings_in_plugins = self._find_template('settings.html')
        if os.path.exists(os.path.join(paths["project"], 'settings.html')):
            settings_in_plugins["project"] = paths["project"]

        for plg in settings_in_plugins:
            settings_path = settings_in_plugins[plg]
            environ_settings = Environment(loader=FileSystemLoader(settings_path))
            tmpl = environ_settings.get_template('settings.html')
            data = {}
            if self.settings_config is not None:
                #this is needed for when the plugins come through bower
                if "fieldtrip-" in plg:
                    plg = plg.replace("fieldtrip-", "")
                if plg in self.settings_config.keys():
                    value = self.settings_config[plg]
                    if value.startswith('{'):
                        data = json.loads(value, object_pairs_hook=collections.OrderedDict)
                    else:
                        data = value
            settings.append(tmpl.render(settings=data))

        header_template = environ.get_template("header.html")
        footer_template = environ.get_template("footer.html")
        template  = environ.get_template('settings.html')
        output = template.render(settings="\n".join(settings),
                            config = self.config,
                            header=header_template.render(
                                data=header_data,
                                platform=self.platform),
                            footer=footer_template.render(
                                    data=footer_data,
                                    platform=self.platform))
        self._write_data(os.sep.join((self.export_path, 'settings.html')), self._prettify(output, 2))

    def _generate_templates(self, environ, templates):
        for templ in templates:
            print "TEMPLATE: generating template {0}".format(templates[templ])
            script_template = environ.get_template(templates[templ])
            self._write_data(os.path.join(self.export_path, 'templates', templates[templ]), script_template.render())

    def _get_data(self, path1, filename, path2):
        '''
        get data from different two paths and merge them
        '''
        with open(os.path.join(path1, filename),'r') as f:
            try:
                json_object = json.load(f, object_pairs_hook=collections.OrderedDict)
                return self._do_merge(filename, json_object, path2)
            except ValueError, e:
                print "There was problem with the json file {0}".format(os.path.join(path1, filename))
                sys.exit()

    def _get_letter(self, obj):
        '''
        Get the letter that corresponds to column in a jqm grid view based on the
        number of elements in obj: see http://api.jquerymobile.com/1.3/grid-layout/
        '''

        letter = 'a'
        if len(obj) > 1:
            i = len(obj) - 2
            letter = chr(i + ord('a'))

        return letter

    def _get_header_footer_data(self, templates_path):
        '''
        get header and footer data
        '''
        header_data = self._get_data(templates_path["core"], 'header.json', templates_path["project"])
        footer_data = self._get_data(templates_path["core"], 'footer.json', templates_path["project"])
        for d in templates_path["plugins"]:
            if os.path.exists(os.path.join(d, 'header.json')):
                self._do_merge('header.json', header_data, d)
            if os.path.exists(os.path.join(d, 'footer.json')):
                self._do_merge('footer.json', footer_data, d)
        return header_data, footer_data

    def _get_plugins_templates(self):
        '''
        get a list of directories with templates
        '''
        plugins_list = []
        #check the folders inside the plugins folder and check for templates folders
        for d in os.listdir(os.path.join(self.root, 'plugins')):
            d1 = os.path.join(self.root, 'plugins', d)
            for dire in os.listdir(d1):
                p = os.path.join(d1, dire)
                if os.path.isdir(p) and not dire.startswith("."):
                    tmpl_path = os.path.join(p, "templates")
                    if os.path.exists(tmpl_path):
                        plugins_list.append(tmpl_path)

        #check for plugins inside the project.json of the project folder in case there are bower plugins
        with open(os.path.join(self.src_dir, 'www', 'theme', 'project.json'),'r') as f:
            plgins = json.load(f)["plugins"]
            for k, v in plgins["fieldtrip"].iteritems():
                if re.match('^v?\d+.\d+.\d+', v):
                    tmpl_path = os.path.join('bower_components', 'fieldtrip-{0}'.format(k), 'src', 'templates')
                    if os.path.exists(tmpl_path):
                        plugins_list.append(os.path.join('bower_components', 'fieldtrip-{0}'.format(k), 'src', 'templates'))
        return plugins_list

    def _is_empty(self, any_structure):
        # TODO
        if any_structure:
            return False
        else:
            return True

    def _is_valid_file(self, f):
        return f.endswith("json") and not f in ["header.json", "footer.json", "settings.json"]

    def _merge(self, a, b, path=None):
        '''
        merges b into a
        #http://stackoverflow.com/questions/7204805/python-dictionaries-of-dictionaries-merge
        '''
        if path is None: path = []
        for key in b:
            if key in a:
                if isinstance(a[key], dict) and isinstance(b[key], dict):
                    if self._is_empty(b[key]):
                        a[key] = b[key]
                    else:
                        self._merge(a[key], b[key], path + [str(key)])
                elif a[key] == b[key]:
                    pass # same leaf value
                else:
                    a[key] = b[key]
            else:
                a[key] = b[key]
        return a

    def _prettify(self, output, indent='2'):
        '''
        custom indentation for BeautifulSoup
        '''
        soup = BeautifulSoup(output, "html5lib")
        if len(soup.findAll('script')) > 0:
            s = soup.prettify()
        else:
            s = soup.div.prettify()
        r = re.compile(r'^(\s*)', re.MULTILINE)
        return r.sub(r'\1\1', s)

    def _sorted(self, dic):
        '''
        sort letters
        dic --> dictionary
        '''
        dic = collections.OrderedDict(sorted(dic.items(), key=lambda t: t[0]))
        return dic

    def _write_data(self, fil, filedata):
        '''
        fil --> filename
        filedata --> content that will be written in filename
        '''
        f = open(fil, 'w')
        f.write(filedata.encode('utf-8'))
        f.close()
<|endoftext|>


<commit_before>from scipy.signal import convolve2d
import graphics
import random
import driver
import game
import time
import sys

rows = 15
cols = 112

class Pong(game.Game):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.lpaddle = graphics.Rectangle(x=0, y=5, width=1, height=5)
        self.rpaddle = graphics.Rectangle(x=111, y=5, width=1, height=5)
        self.ball = graphics.Circle(x=56, y=7, radius=1)
        self.sprites.add(self.lpaddle)
        self.sprites.add(self.rpaddle)
        self.ball.yv = -1
        self.ball.xv = -1
        self.rscore = 0
        self.lscore = 0
        self.leftScore = graphics.TextSprite("0", x=20, y=0, width=5, height=7)
        self.rightScore = graphics.TextSprite("0", x=87, y=0, width=5, height=7)
        self.sprites.add(self.leftScore)
        self.sprites.add(self.rightScore)
        self.leftVictory = graphics.TextSprite("LEFT WINS!", x=13, y=4, width=5, height=7)
        self.rightVictory = graphics.TextSprite("RIGHT WINS!", x=13, y=4, width=5, height=7)
        self.sprites.add(self.ball)
        self.end = False

    def loop(self):
        if self.end:
            time.sleep(3)
            self.sprites = set()
            self.sprites.add(self.lpaddle)
            self.sprites.add(self.rpaddle)
            self.sprites.add(self.leftScore)
            self.sprites.add(self.rightScore)
            self.sprites.add(self.ball)
            self.rscore = 0
            self.lscore = 0
            self.leftScore.set_text("0")
            self.rightScore.set_text("0")
            self.end = False
            self.ball.x = 56
            self.ball.y = 7
            self.ball.xv = -1
            self.ball.yv = -1
        if 'w' in self.keys:
            self.lpaddle.y += 1
            if self.lpaddle.y > 11:
                 self.lpaddle.y = 11
        if 's' in self.keys:
            self.lpaddle.y -= 1
            if self.lpaddle.y < 0:
                 self.lpaddle.y = 0
        if 'o' in self.keys:
            self.rpaddle.y += 1
            if self.rpaddle.y > 11:
                 self.rpaddle.y = 11
        if 'l' in self.keys:
            self.rpaddle.y -= 1
            if self.rpaddle.y < 0:
                 self.rpaddle.y = 0
        if (self.ball.y < (self.ball.radius)) or (self.ball.y > (15-self.ball.radius)):
            self.ball.yv = -1*self.ball.yv
            self.ball.y += self.ball.yv
        if self.ball.x < 1:
          if ((self.ball.y < self.lpaddle.y) or (self.ball.y > (self.lpaddle.y + 4))):
            print(self.ball.y, self.lpaddle.y)
            self.ball.y = 7
            self.ball.x = 56
            self.ball.yv = -1
            self.ball.xv = 1
            self.rscore += 1
            self.rightScore.set_text(str(self.rscore))
            if self.rscore == 10:
                self.sprites = set()
                self.sprites.add(self.rightVictory)
                self.end = True
          else:
            self.ball.xv = -1*self.ball.xv
            self.ball.x += self.ball.xv
        if self.ball.x > 110:
          if ((self.ball.y < self.rpaddle.y) or (self.ball.y > (self.rpaddle.y + 4))):
            print(self.ball.y, self.rpaddle.y)
            self.ball.y = 7
            self.ball.x = 56
            self.ball.yv = -1
            self.ball.xv = -1
            self.lscore += 1
            self.leftScore.set_text(str(self.lscore))
            if self.lscore == 10:
                self.sprites = set()
                self.sprites.add(self.leftVictory)
                self.end = True
          else:
            self.ball.xv = -1*self.ball.xv
            self.ball.x += self.ball.xv

        self.ball.x += self.ball.xv
        self.ball.y += self.ball.yv
        super().loop()

GAME = Pong
<commit_msg>Removed extraneous print statements<commit_after>from scipy.signal import convolve2d
import graphics
import random
import driver
import game
import time
import sys

rows = 15
cols = 112

class Pong(game.Game):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.lpaddle = graphics.Rectangle(x=0, y=5, width=1, height=5)
        self.rpaddle = graphics.Rectangle(x=111, y=5, width=1, height=5)
        self.ball = graphics.Circle(x=56, y=7, radius=1)
        self.sprites.add(self.lpaddle)
        self.sprites.add(self.rpaddle)
        self.ball.yv = -1
        self.ball.xv = -1
        self.rscore = 0
        self.lscore = 0
        self.leftScore = graphics.TextSprite("0", x=20, y=0, width=5, height=7)
        self.rightScore = graphics.TextSprite("0", x=87, y=0, width=5, height=7)
        self.sprites.add(self.leftScore)
        self.sprites.add(self.rightScore)
        self.leftVictory = graphics.TextSprite("LEFT WINS!", x=13, y=4, width=5, height=7)
        self.rightVictory = graphics.TextSprite("RIGHT WINS!", x=13, y=4, width=5, height=7)
        self.sprites.add(self.ball)
        self.end = False

    def loop(self):
        if self.end:
            time.sleep(3)
            self.sprites = set()
            self.sprites.add(self.lpaddle)
            self.sprites.add(self.rpaddle)
            self.sprites.add(self.leftScore)
            self.sprites.add(self.rightScore)
            self.sprites.add(self.ball)
            self.rscore = 0
            self.lscore = 0
            self.leftScore.set_text("0")
            self.rightScore.set_text("0")
            self.end = False
            self.ball.x = 56
            self.ball.y = 7
            self.ball.xv = -1
            self.ball.yv = -1
        if 'w' in self.keys:
            self.lpaddle.y += 1
            if self.lpaddle.y > 11:
                 self.lpaddle.y = 11
        if 's' in self.keys:
            self.lpaddle.y -= 1
            if self.lpaddle.y < 0:
                 self.lpaddle.y = 0
        if 'o' in self.keys:
            self.rpaddle.y += 1
            if self.rpaddle.y > 11:
                 self.rpaddle.y = 11
        if 'l' in self.keys:
            self.rpaddle.y -= 1
            if self.rpaddle.y < 0:
                 self.rpaddle.y = 0
        if (self.ball.y < (self.ball.radius)) or (self.ball.y > (15-self.ball.radius)):
            self.ball.yv = -1*self.ball.yv
            self.ball.y += self.ball.yv
        if self.ball.x < 1:
          if ((self.ball.y < self.lpaddle.y) or (self.ball.y > (self.lpaddle.y + 4))):
            self.ball.y = 7
            self.ball.x = 56
            self.ball.yv = -1
            self.ball.xv = 1
            self.rscore += 1
            self.rightScore.set_text(str(self.rscore))
            if self.rscore == 10:
                self.sprites = set()
                self.sprites.add(self.rightVictory)
                self.end = True
          else:
            self.ball.xv = -1*self.ball.xv
            self.ball.x += self.ball.xv
        if self.ball.x > 110:
          if ((self.ball.y < self.rpaddle.y) or (self.ball.y > (self.rpaddle.y + 4))):
            self.ball.y = 7
            self.ball.x = 56
            self.ball.yv = -1
            self.ball.xv = -1
            self.lscore += 1
            self.leftScore.set_text(str(self.lscore))
            if self.lscore == 10:
                self.sprites = set()
                self.sprites.add(self.leftVictory)
                self.end = True
          else:
            self.ball.xv = -1*self.ball.xv
            self.ball.x += self.ball.xv

        self.ball.x += self.ball.xv
        self.ball.y += self.ball.yv
        super().loop()

GAME = Pong
<|endoftext|>


<commit_before>import yaml
from textwrap import dedent
from jsonschema import validate, ValidationError
import pyaml
import json
import os
import glob
import subprocess
from colorama import init, Fore, Back, Style
from trackhub import Track, default_hub, CompositeTrack, ViewTrack
import pkg_resources
from hubward import utils
from hubward.log import log
import conda.fetch

HERE = os.path.abspath(os.path.dirname(__file__))
SCHEMA = json.loads(utils.get_resource('schema.json'))

class Data(object):
    def __init__(self, obj, reldir):
        """
        Represents a single track destined for upload to UCSC as a track hub.

        In the metadata.yaml file, this is a single entry in the `data` list.

        Parameters
        ----------
        obj : dict
            One entry from the `data` list in a metadata file.

        reldir : str
            The dirname of the metadata file.  All paths (specifically,
            "original", "processed", and "script", are assumed to be relative
            to to `reldir`.
        """
        self.obj = obj
        self.reldir = reldir
        self.original = os.path.join(reldir, obj['original'])
        self.source = obj['source']
        self.processed = os.path.join(reldir, obj['processed'])
        self.description = obj['description']
        self.label = obj['label']
        self.type_ = obj['type']
        self.genome = obj['genome']
        self.script = os.path.join(reldir, obj['script'])
        try:
            self.trackinfo = obj['trackinfo']
        except KeyError:
            self.trackinfo = {}

    def __str__(self):
        return pyaml.dumps(self.obj)

    def needs_download(self):
        if not os.path.exists(self.original):
            return True

    def needs_update(self):
        if not os.path.exists(self.processed):
            return True
        if (

            # if processed is a link, then check the LINK time
            utils.link_is_newer(self.script, self.processed) or

            (os.path.exists(self.original) and
                # but for the original data, we want to FOLLOW the link
                utils.is_newer(self.original, self.processed))
        ):
            return True

    def process(self):
        """
        Runs the processing script, which must accept 2 args, the absolute path
        of the original file and absolute path of the processed file.

        It's up to the processing script to handle everything else.
        """
        cmds = [
            self.script,
            self.original,
            self.processed]
        if not os.path.exists(self.original):
            url = self.source['url']
            log(
                'Downloading "%s" -> "%s"' %
                (url, self.reldir), indent=4)
            fn = os.path.join(self.reldir, 'raw-data', self.source['fn'])
            utils.download(url, fn)
            utils.unpack(fn, os.path.join(self.reldir, 'raw-data'))

        if not os.path.exists(self.original):
            raise ValueError(
                "Original data file %s does not exist" % self.original)
        if not os.path.exists(self.script):
            raise ValueError(
                "Script %s does not exist" % self.script)
        p = subprocess.Popen(
            cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        try:
            stdout, stderr = p.communicate()
        except OSError:
            print "Commands were", ' '.join(cmds)
            raise

        if p.returncode:
            msg = 'CMDS: %s' % (' '.join(cmds))
            msg += '\n'
            msg += 'STDERR: %s' % stderr
            raise ValueError(msg)

        # Check again to make sure we updated the file we thought we did...
        if self.needs_update():
            print(
                Fore.RED + 'The following command did not update '
                '{1}:\n\n{0}\n'.format(' \\\n'.join(cmds), self.processed) +
                Fore.RESET
            )

class Study(object):
    def __init__(self, fn):
        """
        Represents a study documented in a single JSON or YAML file, with
        dictionary keys mapped to easier-to-access attributes.

        Validates against "schema.json" upon loading.

        Parameters
        ----------

        fn : JSON or YAML format file.

        """
        ext = os.path.splitext(fn)[-1]
        if ext == '.json':
            self.metadata = json.load(open(fn))
        elif ext == '.yaml':
            self.metadata = yaml.load(open(fn))

        # validation happens here...
        validate(self.metadata, SCHEMA)

        # Create easier-to-access attributes from the dict sub-keys.
        self.filename = fn
        self.pmid = self.metadata['study']['PMID']
        self.description = self.metadata['study']['description']
        self.reference = self.metadata['study']['reference']
        self.label = self.metadata['study']['label']
        self.sanitized_label = utils.sanitize(self.label, strict=True)
        self.dirname = os.path.abspath(os.path.dirname(fn))
        self.processing = self.metadata['study']['processing']

        self.data = [
            Data(d, os.path.dirname(fn)) for d in self.metadata['data']]

    def __str__(self):
        return pyaml.dumps(self.json)

    def build_metadata(self):
        """
        Call the metadata-builder.py script to reconstruct the metadata.
        """
        dirname = os.path.dirname(self.filename)
        cmds = ['cd', dirname, '&&', 'python', 'metadata-builder.py']
        os.system(' '.join(cmds))

    def process(self, force=False):
        """
        Process all tracks in the study, calling their assigned conversion
        script if data needs update. Report either way.
        """
        log('Study: {0.description}, in "{0.dirname}"'.format(self),
            style=Fore.BLUE)
        for d in self.data:
            if d.needs_update() or force:
                log(
                    'Converting "%s" -> "%s"' %
                    (os.path.relpath(d.original, d.reldir),
                     os.path.relpath(d.processed, d.reldir),
                     ),
                    indent=4)

                d.process()
            else:
                log(
                    'Up to date: "%s"' %
                    os.path.relpath(d.processed, d.reldir), style=Style.DIM,
                    indent=4)
                continue

    def reference_section(self):
        """
        Creates a ReST-formatted reference section to be appended to the end of
        the documentation for the composite track config page.
        """
        if not (self.reference or self.pmid):
            return ""
        if not self.reference:
            reference = ""
        else:
            reference = self.reference
        if not self.pmid:
            pmid = ""
        else:
            pmid = 'http://www.ncbi.nlm.nih.gov/pubmed/{0}'.format(self.pmid)
        return dedent(
            """
            Reference
            ---------
            {0}

            {1}
            """).format(reference, pmid)

    def composite_track(self):
        """
        Create a composite track ready to be added to a trackhub.TrackDb
        instance.
        """
        bigwigs = [i for i in self.data if i.type_ == 'bigwig']
        bigbeds = [i for i in self.data if i.type_ == 'bigbed']

        # Build the HTML docs
        last_section = self.reference_section()
        html_string = utils.reST_to_html(self.processing + last_section)

        composite = CompositeTrack(
            name=utils.sanitize(self.label, strict=True),
            short_label=self.description,
            long_label=self.description,
            html_string=html_string,
            tracktype='bigBed')

        # If there are any bigWigs defined for this study, make a new "signal"
        # subtrack in the composite and then add the bigWigs to it.
        #
        # Use the sanitized label for the study to ensure uniqueness among
        # tracks.
        if len(bigwigs) > 0:
            signal_view = ViewTrack(
                name=self.sanitized_label + 'signalviewtrack',
                view=self.sanitized_label + 'signalview',
                short_label=self.label + ' signal view',
                long_label=self.label + ' signal view',
                visibility='full',
                maxHeightPixels='100:25:8',
                autoScale='off',
                tracktype='bigWig',
            )
            composite.add_view(signal_view)

            for bigwig in bigwigs:

                # If the metadata for this track defined a trackinfo section,
                # interpret it as kwargs to trackhub.Track
                try:
                    kwargs = bigwig.trackinfo
                except AttributeError:
                    kwargs = {}
                kwargs = dict((k, str(v)) for k, v in kwargs.items())
                signal_view.add_tracks(
                    Track(
                        # Tracks are named after both study and label to
                        # hopefully ensure uniqueness across the entire hub
                        name=self.sanitized_label + utils.sanitize(bigwig.label),
                        short_label=bigwig.label,
                        long_label=bigwig.description,
                        local_fn=bigwig.processed,
                        tracktype='bigWig',
                        **kwargs
                    )
                )

        # Same thing with bigBeds
        if len(bigbeds) > 0:
            bed_view = ViewTrack(
                name=self.sanitized_label + 'bedviewtrack',
                view=self.sanitized_label + 'bed_view',
                short_label=self.label + ' bed view',
                long_label=self.label + ' bed view',
                visibility='dense',
            )
            composite.add_view(bed_view)
            for bigbed in bigbeds:
                try:
                    kwargs = bigbed.trackinfo
                except AttributeError:
                    kwargs = {}
                track_kwargs = dict(
                        name=self.sanitized_label + utils.sanitize(bigbed.label),
                        short_label=bigbed.label,
                        long_label=bigbed.description,
                        local_fn=bigbed.processed,
                        tracktype='bigBed 9'
                )
                track_kwargs.update(**kwargs)

                bed_view.add_tracks(Track(**track_kwargs))

        return composite

class Group(object):
    def __init__(self, directory, assembly):
        """
        Represents a group of studies, each of which is in a subdirectory of
        provided `directory`.
        """
        self.studies = []
        for metadata in glob.glob(os.path.join(directory, assembly, '*', 'metadata.yaml')):
            self.studies.append(Study(metadata))
<commit_msg>only run metadata-builder.py if it exists;<commit_after>import yaml
from textwrap import dedent
from jsonschema import validate, ValidationError
import pyaml
import json
import os
import glob
import subprocess
from colorama import init, Fore, Back, Style
from trackhub import Track, default_hub, CompositeTrack, ViewTrack
import pkg_resources
from hubward import utils
from hubward.log import log
import conda.fetch

HERE = os.path.abspath(os.path.dirname(__file__))
SCHEMA = json.loads(utils.get_resource('schema.json'))

class Data(object):
    def __init__(self, obj, reldir):
        """
        Represents a single track destined for upload to UCSC as a track hub.

        In the metadata.yaml file, this is a single entry in the `data` list.

        Parameters
        ----------
        obj : dict
            One entry from the `data` list in a metadata file.

        reldir : str
            The dirname of the metadata file.  All paths (specifically,
            "original", "processed", and "script", are assumed to be relative
            to to `reldir`.
        """
        self.obj = obj
        self.reldir = reldir
        self.original = os.path.join(reldir, obj['original'])
        self.source = obj['source']
        self.processed = os.path.join(reldir, obj['processed'])
        self.description = obj['description']
        self.label = obj['label']
        self.type_ = obj['type']
        self.genome = obj['genome']
        self.script = os.path.join(reldir, obj['script'])
        try:
            self.trackinfo = obj['trackinfo']
        except KeyError:
            self.trackinfo = {}

    def __str__(self):
        return pyaml.dumps(self.obj)

    def needs_download(self):
        if not os.path.exists(self.original):
            return True

    def needs_update(self):
        if not os.path.exists(self.processed):
            return True
        if (

            # if processed is a link, then check the LINK time
            utils.link_is_newer(self.script, self.processed) or

            (os.path.exists(self.original) and
                # but for the original data, we want to FOLLOW the link
                utils.is_newer(self.original, self.processed))
        ):
            return True

    def process(self):
        """
        Runs the processing script, which must accept 2 args, the absolute path
        of the original file and absolute path of the processed file.

        It's up to the processing script to handle everything else.
        """
        cmds = [
            self.script,
            self.original,
            self.processed]
        if not os.path.exists(self.original):
            url = self.source['url']
            log(
                'Downloading "%s" -> "%s"' %
                (url, self.reldir), indent=4)
            fn = os.path.join(self.reldir, 'raw-data', self.source['fn'])
            utils.download(url, fn)
            utils.unpack(fn, os.path.join(self.reldir, 'raw-data'))

        if not os.path.exists(self.original):
            raise ValueError(
                "Original data file %s does not exist" % self.original)
        if not os.path.exists(self.script):
            raise ValueError(
                "Script %s does not exist" % self.script)
        p = subprocess.Popen(
            cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        try:
            stdout, stderr = p.communicate()
        except OSError:
            print "Commands were", ' '.join(cmds)
            raise

        if p.returncode:
            msg = 'CMDS: %s' % (' '.join(cmds))
            msg += '\n'
            msg += 'STDERR: %s' % stderr
            raise ValueError(msg)

        # Check again to make sure we updated the file we thought we did...
        if self.needs_update():
            print(
                Fore.RED + 'The following command did not update '
                '{1}:\n\n{0}\n'.format(' \\\n'.join(cmds), self.processed) +
                Fore.RESET
            )

class Study(object):
    def __init__(self, fn):
        """
        Represents a study documented in a single JSON or YAML file, with
        dictionary keys mapped to easier-to-access attributes.

        Validates against "schema.json" upon loading.

        Parameters
        ----------

        fn : JSON or YAML format file.

        """
        ext = os.path.splitext(fn)[-1]
        if ext == '.json':
            self.metadata = json.load(open(fn))
        elif ext == '.yaml':
            self.metadata = yaml.load(open(fn))

        # validation happens here...
        validate(self.metadata, SCHEMA)

        # Create easier-to-access attributes from the dict sub-keys.
        self.filename = fn
        self.pmid = self.metadata['study']['PMID']
        self.description = self.metadata['study']['description']
        self.reference = self.metadata['study']['reference']
        self.label = self.metadata['study']['label']
        self.sanitized_label = utils.sanitize(self.label, strict=True)
        self.dirname = os.path.abspath(os.path.dirname(fn))
        self.processing = self.metadata['study']['processing']

        self.data = [
            Data(d, os.path.dirname(fn)) for d in self.metadata['data']]

    def __str__(self):
        return pyaml.dumps(self.json)

    def build_metadata(self):
        """
        Call the metadata-builder.py script to reconstruct the metadata.
        """
        dirname = os.path.dirname(self.filename)
        if os.path.exists(os.path.join(dirname, 'metadata-builder.py')):
            cmds = ['cd', dirname, '&&', 'python', 'metadata-builder.py']
            os.system(' '.join(cmds))

    def process(self, force=False):
        """
        Process all tracks in the study, calling their assigned conversion
        script if data needs update. Report either way.
        """
        log('Study: {0.description}, in "{0.dirname}"'.format(self),
            style=Fore.BLUE)
        for d in self.data:
            if d.needs_update() or force:
                log(
                    'Converting "%s" -> "%s"' %
                    (os.path.relpath(d.original, d.reldir),
                     os.path.relpath(d.processed, d.reldir),
                     ),
                    indent=4)

                d.process()
            else:
                log(
                    'Up to date: "%s"' %
                    os.path.relpath(d.processed, d.reldir), style=Style.DIM,
                    indent=4)
                continue

    def reference_section(self):
        """
        Creates a ReST-formatted reference section to be appended to the end of
        the documentation for the composite track config page.
        """
        if not (self.reference or self.pmid):
            return ""
        if not self.reference:
            reference = ""
        else:
            reference = self.reference
        if not self.pmid:
            pmid = ""
        else:
            pmid = 'http://www.ncbi.nlm.nih.gov/pubmed/{0}'.format(self.pmid)
        return dedent(
            """
            Reference
            ---------
            {0}

            {1}
            """).format(reference, pmid)

    def composite_track(self):
        """
        Create a composite track ready to be added to a trackhub.TrackDb
        instance.
        """
        bigwigs = [i for i in self.data if i.type_ == 'bigwig']
        bigbeds = [i for i in self.data if i.type_ == 'bigbed']

        # Build the HTML docs
        last_section = self.reference_section()
        html_string = utils.reST_to_html(self.processing + last_section)

        composite = CompositeTrack(
            name=utils.sanitize(self.label, strict=True),
            short_label=self.description,
            long_label=self.description,
            html_string=html_string,
            tracktype='bigBed')

        # If there are any bigWigs defined for this study, make a new "signal"
        # subtrack in the composite and then add the bigWigs to it.
        #
        # Use the sanitized label for the study to ensure uniqueness among
        # tracks.
        if len(bigwigs) > 0:
            signal_view = ViewTrack(
                name=self.sanitized_label + 'signalviewtrack',
                view=self.sanitized_label + 'signalview',
                short_label=self.label + ' signal view',
                long_label=self.label + ' signal view',
                visibility='full',
                maxHeightPixels='100:25:8',
                autoScale='off',
                tracktype='bigWig',
            )
            composite.add_view(signal_view)

            for bigwig in bigwigs:

                # If the metadata for this track defined a trackinfo section,
                # interpret it as kwargs to trackhub.Track
                try:
                    kwargs = bigwig.trackinfo
                except AttributeError:
                    kwargs = {}
                kwargs = dict((k, str(v)) for k, v in kwargs.items())
                signal_view.add_tracks(
                    Track(
                        # Tracks are named after both study and label to
                        # hopefully ensure uniqueness across the entire hub
                        name=self.sanitized_label + utils.sanitize(bigwig.label),
                        short_label=bigwig.label,
                        long_label=bigwig.description,
                        local_fn=bigwig.processed,
                        tracktype='bigWig',
                        **kwargs
                    )
                )

        # Same thing with bigBeds
        if len(bigbeds) > 0:
            bed_view = ViewTrack(
                name=self.sanitized_label + 'bedviewtrack',
                view=self.sanitized_label + 'bed_view',
                short_label=self.label + ' bed view',
                long_label=self.label + ' bed view',
                visibility='dense',
            )
            composite.add_view(bed_view)
            for bigbed in bigbeds:
                try:
                    kwargs = bigbed.trackinfo
                except AttributeError:
                    kwargs = {}
                track_kwargs = dict(
                        name=self.sanitized_label + utils.sanitize(bigbed.label),
                        short_label=bigbed.label,
                        long_label=bigbed.description,
                        local_fn=bigbed.processed,
                        tracktype='bigBed 9'
                )
                track_kwargs.update(**kwargs)

                bed_view.add_tracks(Track(**track_kwargs))

        return composite

class Group(object):
    def __init__(self, directory, assembly):
        """
        Represents a group of studies, each of which is in a subdirectory of
        provided `directory`.
        """
        self.studies = []
        for metadata in glob.glob(os.path.join(directory, assembly, '*', 'metadata.yaml')):
            self.studies.append(Study(metadata))
<|endoftext|>