<commit_before>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

__author__ = 'elip'

import requests
from neutronclient.common.exceptions import NeutronException
from novaclient.exceptions import NotFound

from cosmo_tester.framework.util import YamlPatcher
from cosmo_tester.framework.testenv import TestCase
from cosmo_tester.framework.util import get_yaml_as_dict
from cosmo_tester.framework.openstack_api import openstack_clients
from cosmo_tester.framework.git_helper import clone

class HelloWorldBashTest(TestCase):

    CLOUDIFY_EXAMPLES_URL = "https://github.com/cloudify-cosmo/" \
                            "cloudify-examples.git"

    def test_hello_world_bash(self):
        self.repo_dir = clone(self.CLOUDIFY_EXAMPLES_URL, self.workdir)
        self.blueprint_path = self.repo_dir / 'hello-world'
        self.blueprint_yaml = self.blueprint_path / 'blueprint.yaml'
        self.modify_yaml(env=self.env,
                         yaml_file=self.blueprint_yaml,
                         host_name='bash-web-server',
                         security_groups=['webserver_security_group'])

        self.upload_deploy_and_execute_install(fetch_state=False)

        (floatingip_node,
         security_group_node,
         server_node) = self.get_instances(client=self.client,
                                           deployment_id=self.test_id)

        self.verify_webserver_running(blueprint_yaml=self.blueprint_yaml,
                                      floatingip_node=floatingip_node)

        nova, neutron = openstack_clients(self.env.cloudify_config)

        server_id = server_node.runtime_properties['openstack_server_id']
        floating_ip_id = floatingip_node.runtime_properties['external_id']
        sg_id = security_group_node.runtime_properties['external_id']

        nova_server = nova.servers.get(server_id)
        neutron_floating_ip = neutron.show_floatingip(floating_ip_id)
        neutron_sg = neutron.show_security_group(sg_id)
        self.logger.info("Agent server : {0}".format(nova_server))
        self.logger.info("Floating ip : {0}".format(neutron_floating_ip))
        self.logger.info("Agent security group : {0}".format(neutron_sg))

        self.execute_uninstall()
        # No components should exist after uninstall
        self.assertRaises(NotFound, nova.servers.get, server_id)
        self.assertRaises(NeutronException, neutron.show_security_group, sg_id)
        self.assertRaises(NeutronException, neutron.show_floatingip,
                          floating_ip_id)

    @staticmethod
    def verify_webserver_running(blueprint_yaml, floatingip_node):
        blueprint = get_yaml_as_dict(blueprint_yaml)
        webserver_props = blueprint['blueprint']['nodes'][3]['properties']
        server_port = webserver_props['port']
        server_ip = floatingip_node.runtime_properties['floating_ip_address']
        server_response = requests.get('http://{0}:{1}'.format(server_ip,
                                                               server_port))
        if server_response.status_code != 200:
            raise AssertionError('Unexpected status code: {}'
                                 .format(server_response.status_code))

    @staticmethod
    def get_instances(client, deployment_id):
        server_node = None
        security_group_node = None
        floatingip_node = None
        instances = client.node_instances.list(
            deployment_id=deployment_id)
        for instance in instances:
            if instance.node_id == 'virtual_ip':
                floatingip_node = instance
            if instance.node_id == 'security_group':
                security_group_node = instance
            if instance.node_id == 'vm':
                server_node = instance
        return floatingip_node, security_group_node, server_node

    @staticmethod
    def modify_yaml(env, yaml_file, host_name, security_groups,
                    security_group_name=None):
        with YamlPatcher(yaml_file) as patch:
            vm_properties_path = 'blueprint.nodes[2].properties'
            patch.merge_obj('{0}.server'.format(vm_properties_path), {
                'name': host_name,
                'image_name': env.ubuntu_image_name,
                'flavor_name': env.flavor_name,
                'security_groups': security_groups,
            })
            sg_name_path = 'blueprint.nodes[1].properties.security_group.name'
            if security_group_name is not None:
                patch.set_value(sg_name_path, security_group_name)
<commit_msg>CFY-861 some docs<commit_after>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

__author__ = 'elip'

import requests
from neutronclient.common.exceptions import NeutronException
from novaclient.exceptions import NotFound

from cosmo_tester.framework.util import YamlPatcher
from cosmo_tester.framework.testenv import TestCase
from cosmo_tester.framework.util import get_yaml_as_dict
from cosmo_tester.framework.openstack_api import openstack_clients
from cosmo_tester.framework.git_helper import clone

class HelloWorldBashTest(TestCase):

    CLOUDIFY_EXAMPLES_URL = "https://github.com/cloudify-cosmo/" \
                            "cloudify-examples.git"

    def test_hello_world_bash(self):
        self.repo_dir = clone(self.CLOUDIFY_EXAMPLES_URL, self.workdir)
        self.blueprint_path = self.repo_dir / 'hello-world'
        self.blueprint_yaml = self.blueprint_path / 'blueprint.yaml'
        self.modify_yaml(env=self.env,
                         yaml_file=self.blueprint_yaml,
                         host_name='bash-web-server',
                         security_groups=['webserver_security_group'])

        self.upload_deploy_and_execute_install(fetch_state=False)

        (floatingip_node,
         security_group_node,
         server_node) = self.get_instances(client=self.client,
                                           deployment_id=self.test_id)

        self.verify_webserver_running(blueprint_yaml=self.blueprint_yaml,
                                      floatingip_node=floatingip_node)

        nova, neutron = openstack_clients(self.env.cloudify_config)

        server_id = server_node.runtime_properties['openstack_server_id']
        floating_ip_id = floatingip_node.runtime_properties['external_id']
        sg_id = security_group_node.runtime_properties['external_id']

        nova_server = nova.servers.get(server_id)
        neutron_floating_ip = neutron.show_floatingip(floating_ip_id)
        neutron_sg = neutron.show_security_group(sg_id)
        self.logger.info("Agent server : {0}".format(nova_server))
        self.logger.info("Floating ip : {0}".format(neutron_floating_ip))
        self.logger.info("Agent security group : {0}".format(neutron_sg))

        self.execute_uninstall()
        # No components should exist after uninstall
        self.assertRaises(NotFound, nova.servers.get, server_id)
        self.assertRaises(NeutronException, neutron.show_security_group, sg_id)
        self.assertRaises(NeutronException, neutron.show_floatingip,
                          floating_ip_id)

    @staticmethod
    def verify_webserver_running(blueprint_yaml, floatingip_node):
        # this method is also used by two_deployments_test
        blueprint = get_yaml_as_dict(blueprint_yaml)
        webserver_props = blueprint['blueprint']['nodes'][3]['properties']
        server_port = webserver_props['port']
        server_ip = floatingip_node.runtime_properties['floating_ip_address']
        server_response = requests.get('http://{0}:{1}'.format(server_ip,
                                                               server_port))
        if server_response.status_code != 200:
            raise AssertionError('Unexpected status code: {}'
                                 .format(server_response.status_code))

    @staticmethod
    def get_instances(client, deployment_id):
        # this method is also used by two_deployments_test
        server_node = None
        security_group_node = None
        floatingip_node = None
        instances = client.node_instances.list(
            deployment_id=deployment_id)
        for instance in instances:
            if instance.node_id == 'virtual_ip':
                floatingip_node = instance
            if instance.node_id == 'security_group':
                security_group_node = instance
            if instance.node_id == 'vm':
                server_node = instance
        return floatingip_node, security_group_node, server_node

    @staticmethod
    def modify_yaml(env, yaml_file, host_name, security_groups,
                    security_group_name=None):
        # this method is also used by two_deployments_test
        with YamlPatcher(yaml_file) as patch:
            vm_properties_path = 'blueprint.nodes[2].properties'
            patch.merge_obj('{0}.server'.format(vm_properties_path), {
                'name': host_name,
                'image_name': env.ubuntu_image_name,
                'flavor_name': env.flavor_name,
                'security_groups': security_groups,
            })
            sg_name_path = 'blueprint.nodes[1].properties.security_group.name'
            if security_group_name is not None:
                patch.set_value(sg_name_path, security_group_name)
<|endoftext|>


<commit_before>import sys
import pathlib
import sqlalchemy as sa
import argparse
import requests

import databot.db
import databot.pipes
from databot.db.models import Models
from databot.db.utils import strip_prefix, get_or_create, get_engine
from databot.db.migrations import Migrations
from databot.printing import Printer
from databot import commands

class Bot(object):

    def __init__(self, uri_or_engine='sqlite:///:memory:', *,
                 debug=False, retry=False, limit=0, verbosity=0, output=sys.stdout, models=None):
        self.path = pathlib.Path(sys.modules[self.__class__.__module__].__file__).resolve().parent
        self.engine = get_engine(uri_or_engine, self.path)
        self.models = models or Models(sa.MetaData(self.engine))
        self.output = Printer(self.models, output)
        self.conn = self.engine.connect()
        self.pipes = []
        self.pipes_by_name = {}
        self.pipes_by_id = {}
        self.stack = []
        self.options = []
        self.name = '%s.%s' % (self.__class__.__module__, self.__class__.__name__)
        self.debug = debug
        self.retry = retry
        self.limit = limit
        self.verbosity = verbosity
        self.download_delay = None
        self.requests = requests.Session()
        self.commands = self._register_commands(commands.CommandsManager(self))

        self.migrations = Migrations(self.models, self.engine, self.output, verbosity=1)
        if self.migrations.has_initial_state():
            self.migrations.initialize()

    def define(self, name, uri_or_engine=None, compress=None):
        """Defines new pipe for storing data.

        Parameters
        ----------
        name : str
            Pipe name.
        uri_or_engine : str or sqlalchemy.Engine
            Database engine if this pipe used external database (other than defined in bot).
        compression : databot.db.models.Compression, optional
            Default compression for data values.
        """
        if name in self.pipes_by_name:
            raise ValueError('A pipe with "%s" name is already defined.' % name)

        if uri_or_engine is not None:
            samedb = False
            engine = get_engine(uri_or_engine, self.path)
            models = Models(sa.MetaData())
            migrations = Migrations(models, engine, self.output, verbosity=1)
            if migrations.has_initial_state():
                migrations.initialize()
            else:
                unapplied_migrations = migrations.unapplied()
                if unapplied_migrations:
                    dburl = uri_or_engine if isinstance(uri_or_engine, str) else engine.url
                    self.output.error('\n'.join([
                        "External database '%s' from '%s' pipe has unapplied migrations.\n" % (dburl, name),
                        'You need to run database migrations:\n',
                        '    databot %s migrate\n' % dburl,
                        "List of unapplied migrations:\n\n  - %s\n" % (
                            '\n  - '.join([f.__name__ for f in unapplied_migrations])
                        ),
                    ]))
                    sys.exit(1)

            defaults = dict(bot=self.name, pipe=name)
            internal = get_or_create(self.engine, self.models.pipes, ('pipe',), defaults)
            external = get_or_create(engine, models.pipes, ('pipe',), defaults)

            table_id = internal.id
            table_name = 't%d' % external.id

        else:
            samedb = True
            engine = self.engine
            models = self.models

            defaults = dict(bot=self.name, pipe=name)
            internal = get_or_create(engine, models.pipes, ('pipe',), defaults)

            table_id = internal.id
            table_name = 't%d' % internal.id

        table = models.get_data_table(table_name)
        table.create(engine, checkfirst=True)

        pipe = databot.pipes.Pipe(self, table_id, name, table, engine, samedb, compress)
        self.pipes.append(pipe)
        self.pipes_by_name[name] = pipe
        self.pipes_by_id[pipe.id] = pipe

        return pipe

    def pipe(self, name):
        return self.pipes_by_name[name]

    def query_retry_pipes(self):
        errors, state, pipes = self.models.errors, self.models.state, self.models.pipes

        error = errors.alias('error')
        source = pipes.alias('source')
        target = pipes.alias('target')

        query = (
            sa.select([error, source, target], use_labels=True).
            select_from(
                error.
                join(state).
                join(source, state.c.source_id == source.c.id).
                join(target, state.c.target_id == target.c.id)
            )
        )

        for row in self.engine.execute(query):
            error = strip_prefix(row, 'error_')
            error['source'] = strip_prefix(row, 'source_')
            error['target'] = strip_prefix(row, 'target_')
            yield error

    def compact(self):
        for pipe in self.pipes:
            pipe.compact()

    def _register_commands(self, cmgr):
        cmgr.register('status', commands.Status)
        cmgr.register('select', commands.Select)
        cmgr.register('download', commands.Download)
        cmgr.register('skip', commands.Skip)
        cmgr.register('reset', commands.Reset)
        cmgr.register('offset', commands.Offset)
        cmgr.register('clean', commands.Clean)
        cmgr.register('compact', commands.Compact)
        cmgr.register('show', commands.Show)
        cmgr.register('tail', commands.Tail)
        cmgr.register('export', commands.Export)
        cmgr.register('resolve', commands.Resolve)
        cmgr.register('migrate', commands.Migrate)
        cmgr.register('errors', commands.Errors)
        cmgr.register('sh', commands.Shell)
        cmgr.register('rename', commands.Rename)
        cmgr.register('compress', commands.Compress)
        cmgr.register('decompress', commands.Decompress)
        return cmgr

    def main(self, define=None, run=None, argv=None):
        parser = argparse.ArgumentParser()

        # Vorbosity levels:
        # 0 - no output
        # 1 - show progress bar
        parser.add_argument('-v', '--verbosity', type=int, default=1)

        sps = parser.add_subparsers(dest='command')

        cmgr = commands.CommandsManager(self, sps)
        cmgr.register('run', commands.Run, run)
        self._register_commands(cmgr)

        args = parser.parse_args(argv)

        self.verbosity = args.verbosity

        if args.command == 'migrate':
            cmgr.run(args.command, args, default='status')
        elif args.command or define:
            self.check_migrations(self.migrations)

            if define is not None:
                define(self)

            try:
                cmgr.run(args.command, args, default='status')
            except KeyboardInterrupt:
                pass

        return self

    def check_migrations(self, migrations):
        unapplied_migrations = migrations.unapplied()
        if unapplied_migrations:
            self.output.error((
                'You need to run database migrations:\n'
                '\n'
                '    %s migrate\n'
                '\n'
                'List of unapplied migrations:\n\n  - %s\n'
            ) % (sys.argv[0], '\n  - '.join([f.__name__ for f in unapplied_migrations])))
            sys.exit(1)

    def run(self, name):
        if self.options:
            return name in self.options
        else:
            return True
<commit_msg>Show better error message when database needs a migration<commit_after>import sys
import pathlib
import sqlalchemy as sa
import argparse
import requests

import databot.db
import databot.pipes
from databot.db.models import Models
from databot.db.utils import strip_prefix, get_or_create, get_engine
from databot.db.migrations import Migrations
from databot.printing import Printer
from databot import commands

class Bot(object):

    def __init__(self, uri_or_engine='sqlite:///:memory:', *,
                 debug=False, retry=False, limit=0, verbosity=0, output=sys.stdout, models=None):
        self.path = pathlib.Path(sys.modules[self.__class__.__module__].__file__).resolve().parent
        self.engine = get_engine(uri_or_engine, self.path)
        self.models = models or Models(sa.MetaData(self.engine))
        self.output = Printer(self.models, output)
        self.conn = self.engine.connect()
        self.pipes = []
        self.pipes_by_name = {}
        self.pipes_by_id = {}
        self.stack = []
        self.options = []
        self.name = '%s.%s' % (self.__class__.__module__, self.__class__.__name__)
        self.debug = debug
        self.retry = retry
        self.limit = limit
        self.verbosity = verbosity
        self.download_delay = None
        self.requests = requests.Session()
        self.commands = self._register_commands(commands.CommandsManager(self))

        self.migrations = Migrations(self.models, self.engine, self.output, verbosity=1)
        if self.migrations.has_initial_state():
            self.migrations.initialize()

    def define(self, name, uri_or_engine=None, compress=None):
        """Defines new pipe for storing data.

        Parameters
        ----------
        name : str
            Pipe name.
        uri_or_engine : str or sqlalchemy.Engine
            Database engine if this pipe used external database (other than defined in bot).
        compression : databot.db.models.Compression, optional
            Default compression for data values.
        """
        if name in self.pipes_by_name:
            raise ValueError('A pipe with "%s" name is already defined.' % name)

        if uri_or_engine is not None:
            samedb = False
            engine = get_engine(uri_or_engine, self.path)
            models = Models(sa.MetaData())
            migrations = Migrations(models, engine, self.output, verbosity=1)
            if migrations.has_initial_state():
                migrations.initialize()
            else:
                unapplied_migrations = migrations.unapplied()
                if unapplied_migrations:
                    dburl = uri_or_engine if isinstance(uri_or_engine, str) else engine.url
                    self.output.error('\n'.join([
                        "External database '%s' from '%s' pipe has unapplied migrations.\n" % (dburl, name),
                        'You need to run database migrations:\n',
                        '    databot %s migrate\n' % dburl,
                        "List of unapplied migrations:\n\n  - %s\n" % (
                            '\n  - '.join([f.__name__ for f in unapplied_migrations])
                        ),
                    ]))
                    sys.exit(1)

            defaults = dict(bot=self.name, pipe=name)
            internal = get_or_create(self.engine, self.models.pipes, ('pipe',), defaults)
            external = get_or_create(engine, models.pipes, ('pipe',), defaults)

            table_id = internal.id
            table_name = 't%d' % external.id

        else:
            samedb = True
            engine = self.engine
            models = self.models

            defaults = dict(bot=self.name, pipe=name)
            internal = get_or_create(engine, models.pipes, ('pipe',), defaults)

            table_id = internal.id
            table_name = 't%d' % internal.id

        table = models.get_data_table(table_name)
        table.create(engine, checkfirst=True)

        pipe = databot.pipes.Pipe(self, table_id, name, table, engine, samedb, compress)
        self.pipes.append(pipe)
        self.pipes_by_name[name] = pipe
        self.pipes_by_id[pipe.id] = pipe

        return pipe

    def pipe(self, name):
        return self.pipes_by_name[name]

    def query_retry_pipes(self):
        errors, state, pipes = self.models.errors, self.models.state, self.models.pipes

        error = errors.alias('error')
        source = pipes.alias('source')
        target = pipes.alias('target')

        query = (
            sa.select([error, source, target], use_labels=True).
            select_from(
                error.
                join(state).
                join(source, state.c.source_id == source.c.id).
                join(target, state.c.target_id == target.c.id)
            )
        )

        for row in self.engine.execute(query):
            error = strip_prefix(row, 'error_')
            error['source'] = strip_prefix(row, 'source_')
            error['target'] = strip_prefix(row, 'target_')
            yield error

    def compact(self):
        for pipe in self.pipes:
            pipe.compact()

    def _register_commands(self, cmgr):
        cmgr.register('status', commands.Status)
        cmgr.register('select', commands.Select)
        cmgr.register('download', commands.Download)
        cmgr.register('skip', commands.Skip)
        cmgr.register('reset', commands.Reset)
        cmgr.register('offset', commands.Offset)
        cmgr.register('clean', commands.Clean)
        cmgr.register('compact', commands.Compact)
        cmgr.register('show', commands.Show)
        cmgr.register('tail', commands.Tail)
        cmgr.register('export', commands.Export)
        cmgr.register('resolve', commands.Resolve)
        cmgr.register('migrate', commands.Migrate)
        cmgr.register('errors', commands.Errors)
        cmgr.register('sh', commands.Shell)
        cmgr.register('rename', commands.Rename)
        cmgr.register('compress', commands.Compress)
        cmgr.register('decompress', commands.Decompress)
        return cmgr

    def main(self, define=None, run=None, argv=None):
        parser = argparse.ArgumentParser()

        # Vorbosity levels:
        # 0 - no output
        # 1 - show progress bar
        parser.add_argument('-v', '--verbosity', type=int, default=1)

        sps = parser.add_subparsers(dest='command')

        cmgr = commands.CommandsManager(self, sps)
        cmgr.register('run', commands.Run, run)
        self._register_commands(cmgr)

        args = parser.parse_args(argv)

        self.verbosity = args.verbosity

        if args.command == 'migrate':
            cmgr.run(args.command, args, default='status')
        elif args.command or define:
            self.check_migrations(self.migrations)

            if define is not None:
                define(self)

            try:
                cmgr.run(args.command, args, default='status')
            except KeyboardInterrupt:
                pass

        return self

    def check_migrations(self, migrations):
        unapplied_migrations = migrations.unapplied()
        if unapplied_migrations:
            self.output.error((
                'You need to run database migrations:\n'
                '\n'
                '    databot %s migrate\n'
                '\n'
                'List of unapplied migrations:\n\n  - %s\n'
            ) % (self.engine.url, '\n  - '.join([f.__name__ for f in unapplied_migrations])))
            sys.exit(1)

    def run(self, name):
        if self.options:
            return name in self.options
        else:
            return True
<|endoftext|>


<commit_before>#!/usr/bin/env python
import argparse
import os
import re
import shutil
import SimpleHTTPServer
import SocketServer
import sys

OUT_DIR = os.path.abspath(os.path.join(".", "build"))
templates = {}

def collect_pages():
    """Collect source page paths from "pages" folder in pwd"""
    pages = []
    for (dirpath, dirnames, filenames) in os.walk("pages/"):
        for filename in filenames:
            if filename.endswith(".html") or filename.endswith(".htm"):
                pages.append(os.path.join(dirpath, filename).split(os.path.sep, 1)[1])
    return pages

def create_out_dir():
    """Create the output dir if it doens't exist"""
    if not os.path.isdir(OUT_DIR):
        os.makedirs(OUT_DIR)

def load_pages(pages):
    """Load pages and extract page variables

    Expects list of file paths
    Returns list of dictionaries of {filename, page_data, [page_vars]}
    """
    loaded_pages = []

    for page in pages:
        with open(os.path.join("pages", page), "r") as page_file:
            page_data = page_file.read()
            (page_data, page_vars) = get_and_strip_page_vars(page_data)
            loaded_pages.append({"filename": page, "page_data": page_data, "page_vars": page_vars})
    return loaded_pages

def check_templates_exist(pages):
    """Checks that all templates exist, exits with error if any are missing"""
    errored = False
    for page in pages:
        if "template" in page["page_vars"]:
            if not os.path.exists(os.path.join(".", "templates", page["page_vars"]["template"])):
                print("ERROR: Template '{0}' not found".format(page["page_vars"]["template"]))
                errored = True
    if errored:
        sys.exit(1)

def render_pages(pages):
    """Render pages to HTML using templates

    Expects list of dictionaries of {filename, page_data, [page_vars]}
    Returns list of dictionaries of {filename, rendered_page_content}
    """
    rendered_pages = []
    for page in pages:
        if "template" in page["page_vars"]:
            template = get_template(page["page_vars"]["template"])
        else:
            template = get_template("base.html")

        rendered_page = template.replace("$page_content", page["page_data"])
        rendered_page = replace_includes(rendered_page)
        for page_var in page["page_vars"]:
            rendered_page = rendered_page.replace("$" + page_var, page["page_vars"][page_var])
        rendered_pages.append({"filename": page["filename"], "content": rendered_page})
    return rendered_pages

def get_template(template_path):
    """Memoized function to get page template.
    Loads templates as neccessary.
    """
    if template_path not in templates:
        with open(os.path.join(".", "templates", template_path), "r") as template_file:
            templates[template_path] = template_file.read()
    return templates[template_path]

def get_and_strip_page_vars(page_data):
    """Returns a tuple of page_data (with page vars stripped) and
    dictionary of <!-- $var = "value" --> pairs from page data
    """
    regex_var_capture = "<!--\s+(\w+)\s+=\s+\"([^>]*)\"\s+-->"
    matches = re.findall(regex_var_capture, page_data)
    page_vars = {}
    for match in matches:
        page_vars[match[0]] = match[1]
        regex_this_var = "<!--\s+(\{0})\s+=\s+\"({1})\"\s+-->".format(match[0], match[1])
        page_data = re.sub(regex_this_var, "", page_data)
    return (page_data, page_vars)

def replace_includes(page_data):
    """Replace <!-- include("my_include.html") --> directives with the
    referenced file"""
    regex_include = "<!--\s+include\(\"([^>]*)\"\)\s+-->"
    matches = re.findall(regex_include, page_data)
    for match in matches:
        include_data = get_include_data(match)
        regex_this_include = "<!--\s+include\(\"{0}\"\)\s+-->".format(match)
        page_data = re.sub(regex_this_include, include_data, page_data)
    return page_data

include_cache = {}
def get_include_data(include_filename):
    """Memoized function to get include file data"""
    if include_filename in include_cache:
        return include_cache[include_filename]

    include_full_filename = os.path.join("templates", "includes", include_filename)
    if not os.path.isfile(include_full_filename):
        print("Error: Included file {0} not found".format(include_filename))
        sys.exit(1)

    with open(include_full_filename, "r") as include_file:
        include_file_data = include_file.read()

    include_cache[include_filename] = include_file_data
    return include_file_data

def write_pages(rendered_pages):
    """Write rendered page data to files in OUT_DIR"""
    for page in rendered_pages:
        if not os.path.isdir(os.path.join(OUT_DIR, os.path.dirname(page["filename"]))):
            os.makedirs(os.path.join(OUT_DIR, os.path.dirname(page["filename"])))
        with open(os.path.join(OUT_DIR, page["filename"]), "w") as out_file:
            out_file.write(page["content"])

def merge_dirs(source_path, dest_path):
    """Copy the contents of source_path to dest_path, creating subdirectories where
    neccessary.

    Will not overwrite contents in dest_path.
    """
    source_path_abs = os.path.abspath(source_path)
    for (dirpath, dirnames, filenames) in os.walk(source_path_abs):
        for filename in filenames:
            source_path_file = os.path.join(dirpath, filename)
            source_path_rel = source_path_file.replace(source_path_abs, "").lstrip(os.path.sep)
            source_path_dir = os.path.dirname(source_path_rel)
            if not os.path.isdir(os.path.join(dest_path, source_path_dir)):
                os.makedirs(os.path.join(dest_path, source_path_dir))
            if not os.path.isfile(os.path.join(dest_path, source_path_rel)):
                shutil.copyfile(source_path_file, os.path.join(dest_path, source_path_rel))

def copy_template_assets():
    """Copy template asset files to output directory"""
    if os.path.isdir(os.path.join("templates", "static")):
        merge_dirs(os.path.join("templates", "static"), os.path.join(OUT_DIR, "static"))

def copy_site_assets():
    """Copy site asset files to output directory"""
    if os.path.isdir(os.path.join(OUT_DIR, "static")):
        merge_dirs("static", os.path.join(OUT_DIR, "static"))

def clean_build_path():
    """Remove build folder if it exists"""
    if os.path.isdir(OUT_DIR):
        shutil.rmtree(OUT_DIR)

def serve_content():
    """Serve rendered content on SimpleHTTPServer. This helps with absolute
    file paths such as /static/css/main.css, which wouldn't work
    properly if opened directly in from file:///"""
    PORT = 8000
    os.chdir(OUT_DIR)
    Handler = SimpleHTTPServer.SimpleHTTPRequestHandler

    # Allow us to quickly kill and restart server without waiting for TCP
    # socket to close down completely
    SocketServer.TCPServer.allow_reuse_address = 1
    httpd = SocketServer.TCPServer(("", PORT), Handler)
    print("Serving content at localhost: " + str(PORT))
    try:
        httpd.serve_forever()
    except KeyboardInterrupt:
        print("Exiting")
        sys.exit()

def check_required_paths():
    """Check that the required bsw project paths exist"""
    template_path = os.path.join(".", "templates", "base.html")
    if not os.path.exists(template_path):
        print("ERROR: base template (templates/base.html) not found")
        print("Did you remember to create the 'templates' folder and 'base.html' template?")
        sys.exit(1)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="bsw - build static website")
    parser.add_argument("-C", "--clean", action="store_true",
                        help="remove existing build folder before building")
    parser.add_argument("-s", "--http-server", action="store_true",
                        help="serve content after build (default port 8000)")
    args = parser.parse_args()

    if args.clean:
        clean_build_path()

    check_required_paths()

    create_out_dir()

    print("Colecting source pages")
    pages = collect_pages()

    print("Loading pages")
    pages_with_data = load_pages(pages)
    check_templates_exist(pages_with_data)

    print("Rendering pages with template")
    rendered_pages = render_pages(pages_with_data)

    print("Writing pages")
    write_pages(rendered_pages)

    print("Copying template assets")
    copy_template_assets()

    print("Copying site assets")
    copy_site_assets()

    print("Static site build complete")

    if args.http_server:
        serve_content()
<commit_msg>Update comment<commit_after>#!/usr/bin/env python
import argparse
import os
import re
import shutil
import SimpleHTTPServer
import SocketServer
import sys

OUT_DIR = os.path.abspath(os.path.join(".", "build"))
templates = {}

def collect_pages():
    """Collect source page paths from "pages" folder in pwd"""
    pages = []
    for (dirpath, dirnames, filenames) in os.walk("pages/"):
        for filename in filenames:
            if filename.endswith(".html") or filename.endswith(".htm"):
                pages.append(os.path.join(dirpath, filename).split(os.path.sep, 1)[1])
    return pages

def create_out_dir():
    """Create the output dir if it doens't exist"""
    if not os.path.isdir(OUT_DIR):
        os.makedirs(OUT_DIR)

def load_pages(pages):
    """Load pages and extract page variables

    Expects list of file paths
    Returns list of dictionaries of {filename, page_data, [page_vars]}
    """
    loaded_pages = []

    for page in pages:
        with open(os.path.join("pages", page), "r") as page_file:
            page_data = page_file.read()
            (page_data, page_vars) = get_and_strip_page_vars(page_data)
            loaded_pages.append({"filename": page, "page_data": page_data, "page_vars": page_vars})
    return loaded_pages

def check_templates_exist(pages):
    """Checks that all templates exist, exits with error if any are missing"""
    errored = False
    for page in pages:
        if "template" in page["page_vars"]:
            if not os.path.exists(os.path.join(".", "templates", page["page_vars"]["template"])):
                print("ERROR: Template '{0}' not found".format(page["page_vars"]["template"]))
                errored = True
    if errored:
        sys.exit(1)

def render_pages(pages):
    """Render pages to HTML using templates

    Expects list of dictionaries of {filename, page_data, [page_vars]}
    Returns list of dictionaries of {filename, rendered_page_content}
    """
    rendered_pages = []
    for page in pages:
        if "template" in page["page_vars"]:
            template = get_template(page["page_vars"]["template"])
        else:
            template = get_template("base.html")

        rendered_page = template.replace("$page_content", page["page_data"])
        rendered_page = replace_includes(rendered_page)
        for page_var in page["page_vars"]:
            rendered_page = rendered_page.replace("$" + page_var, page["page_vars"][page_var])
        rendered_pages.append({"filename": page["filename"], "content": rendered_page})
    return rendered_pages

def get_template(template_path):
    """Memoized function to get page template.
    Loads templates as neccessary.
    """
    if template_path not in templates:
        with open(os.path.join(".", "templates", template_path), "r") as template_file:
            templates[template_path] = template_file.read()
    return templates[template_path]

def get_and_strip_page_vars(page_data):
    """Returns a tuple of (page_data, [page_vars]) where page data
    has had vars stripped and page_vars is a list of dictionaries of
    {var: value}
    """
    regex_var_capture = "<!--\s+(\w+)\s+=\s+\"([^>]*)\"\s+-->"
    matches = re.findall(regex_var_capture, page_data)
    page_vars = {}
    for match in matches:
        page_vars[match[0]] = match[1]
        regex_this_var = "<!--\s+(\{0})\s+=\s+\"({1})\"\s+-->".format(match[0], match[1])
        page_data = re.sub(regex_this_var, "", page_data)
    return (page_data, page_vars)

def replace_includes(page_data):
    """Replace <!-- include("my_include.html") --> directives with the
    referenced file"""
    regex_include = "<!--\s+include\(\"([^>]*)\"\)\s+-->"
    matches = re.findall(regex_include, page_data)
    for match in matches:
        include_data = get_include_data(match)
        regex_this_include = "<!--\s+include\(\"{0}\"\)\s+-->".format(match)
        page_data = re.sub(regex_this_include, include_data, page_data)
    return page_data

include_cache = {}
def get_include_data(include_filename):
    """Memoized function to get include file data"""
    if include_filename in include_cache:
        return include_cache[include_filename]

    include_full_filename = os.path.join("templates", "includes", include_filename)
    if not os.path.isfile(include_full_filename):
        print("Error: Included file {0} not found".format(include_filename))
        sys.exit(1)

    with open(include_full_filename, "r") as include_file:
        include_file_data = include_file.read()

    include_cache[include_filename] = include_file_data
    return include_file_data

def write_pages(rendered_pages):
    """Write rendered page data to files in OUT_DIR"""
    for page in rendered_pages:
        if not os.path.isdir(os.path.join(OUT_DIR, os.path.dirname(page["filename"]))):
            os.makedirs(os.path.join(OUT_DIR, os.path.dirname(page["filename"])))
        with open(os.path.join(OUT_DIR, page["filename"]), "w") as out_file:
            out_file.write(page["content"])

def merge_dirs(source_path, dest_path):
    """Copy the contents of source_path to dest_path, creating subdirectories where
    neccessary.

    Will not overwrite contents in dest_path.
    """
    source_path_abs = os.path.abspath(source_path)
    for (dirpath, dirnames, filenames) in os.walk(source_path_abs):
        for filename in filenames:
            source_path_file = os.path.join(dirpath, filename)
            source_path_rel = source_path_file.replace(source_path_abs, "").lstrip(os.path.sep)
            source_path_dir = os.path.dirname(source_path_rel)
            if not os.path.isdir(os.path.join(dest_path, source_path_dir)):
                os.makedirs(os.path.join(dest_path, source_path_dir))
            if not os.path.isfile(os.path.join(dest_path, source_path_rel)):
                shutil.copyfile(source_path_file, os.path.join(dest_path, source_path_rel))

def copy_template_assets():
    """Copy template asset files to output directory"""
    if os.path.isdir(os.path.join("templates", "static")):
        merge_dirs(os.path.join("templates", "static"), os.path.join(OUT_DIR, "static"))

def copy_site_assets():
    """Copy site asset files to output directory"""
    if os.path.isdir(os.path.join(OUT_DIR, "static")):
        merge_dirs("static", os.path.join(OUT_DIR, "static"))

def clean_build_path():
    """Remove build folder if it exists"""
    if os.path.isdir(OUT_DIR):
        shutil.rmtree(OUT_DIR)

def serve_content():
    """Serve rendered content on SimpleHTTPServer. This helps with absolute
    file paths such as /static/css/main.css, which wouldn't work
    properly if opened directly in from file:///"""
    PORT = 8000
    os.chdir(OUT_DIR)
    Handler = SimpleHTTPServer.SimpleHTTPRequestHandler

    # Allow us to quickly kill and restart server without waiting for TCP
    # socket to close down completely
    SocketServer.TCPServer.allow_reuse_address = 1
    httpd = SocketServer.TCPServer(("", PORT), Handler)
    print("Serving content at localhost: " + str(PORT))
    try:
        httpd.serve_forever()
    except KeyboardInterrupt:
        print("Exiting")
        sys.exit()

def check_required_paths():
    """Check that the required bsw project paths exist"""
    template_path = os.path.join(".", "templates", "base.html")
    if not os.path.exists(template_path):
        print("ERROR: base template (templates/base.html) not found")
        print("Did you remember to create the 'templates' folder and 'base.html' template?")
        sys.exit(1)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="bsw - build static website")
    parser.add_argument("-C", "--clean", action="store_true",
                        help="remove existing build folder before building")
    parser.add_argument("-s", "--http-server", action="store_true",
                        help="serve content after build (default port 8000)")
    args = parser.parse_args()

    if args.clean:
        clean_build_path()

    check_required_paths()

    create_out_dir()

    print("Colecting source pages")
    pages = collect_pages()

    print("Loading pages")
    pages_with_data = load_pages(pages)
    check_templates_exist(pages_with_data)

    print("Rendering pages with template")
    rendered_pages = render_pages(pages_with_data)

    print("Writing pages")
    write_pages(rendered_pages)

    print("Copying template assets")
    copy_template_assets()

    print("Copying site assets")
    copy_site_assets()

    print("Static site build complete")

    if args.http_server:
        serve_content()
<|endoftext|>


<commit_before><commit_msg>Minor output typo<commit_after><|endoftext|>


<commit_before>#    Copyright 2016-2016 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Base matplotlib plotter module"""
from abc import abstractmethod, ABCMeta
from collections import defaultdict as ddict
import matplotlib.pyplot as plt
from trappy.plotter import AttrConf
from trappy.plotter.Constraint import ConstraintManager
from trappy.plotter.PlotLayout import PlotLayout
from trappy.plotter.AbstractDataPlotter import AbstractDataPlotter
from trappy.plotter.ColorMap import ColorMap

class StaticPlot(AbstractDataPlotter):
    """
    This class uses :mod:`trappy.plotter.Constraint.Constraint` to
    represent different permutations of input parameters. These
    constraints are generated by creating an instance of
    :mod:`trappy.plotter.Constraint.ConstraintManager`.

    :param trace: The input data
    :type trace: :mod:`trappy.trace.FTrace` or :mod:`pandas.DataFrame`, list or single

    :param column: specifies the name of the column to
           be plotted.
    :type column: (str, list(str))

    :param templates: TRAPpy events

        .. note::

                This is not required if a :mod:`pandas.DataFrame` is
                used

    :type templates: :mod:`trappy.base.Base`

    :param filters: Filter the column to be plotted as per the
        specified criteria. For Example:
        ::

            filters =
                    {
                        "pid": [ 3338 ],
                        "cpu": [0, 2, 4],
                    }
    :type filters: dict

    :param per_line: Used to control the number of graphs
        in each graph subplot row
    :type per_line: int

    :param concat: Draw all the pivots on a single graph
    :type concat: bool

    :param permute: Draw one plot for each of the traces specified
    :type permute: bool

    :param drawstyle: This argument is forwarded to the matplotlib
        corresponding :func:`matplotlib.pyplot.plot` call

        drawing style.

        .. note::

            step plots are not currently supported for filled
            graphs

    :param xlim: A tuple representing the upper and lower xlimits
    :type xlim: tuple

    :param ylim: A tuple representing the upper and lower ylimits
    :type ylim: tuple

    :param title: A title describing all the generated plots
    :type title: str

    :param style: Created pre-styled graphs loaded from
        :mod:`trappy.plotter.AttrConf.MPL_STYLE`
    :type style: bool

    :param signals: A string of the type event_name:column
        to indicate the value that needs to be plotted

        .. note::

            - Only one of `signals` or both `templates` and
              `columns` should be specified
            - Signals format won't work for :mod:`pandas.DataFrame`
              input

    :type signals: str

    :param legend_ncol: A positive integer that represents the
        number of columns in the legend
    :type legend_ncol: int
    """
    __metaclass__ = ABCMeta

    def __init__(self, traces, templates, **kwargs):
        self._fig = None
        self._layout = None
        super(StaticPlot, self).__init__(traces=traces,
                                         templates=templates)

        self.set_defaults()

        for key in kwargs:
            if key in AttrConf.ARGS_TO_FORWARD:
                self._attr["args_to_forward"][key] = kwargs[key]
            else:
                self._attr[key] = kwargs[key]

        if "signals" in self._attr:
            self._describe_signals()

        self._check_data()

        if "column" not in self._attr:
            raise RuntimeError("Value Column not specified")

        zip_constraints = not self._attr["permute"]
        self.c_mgr = ConstraintManager(traces, self._attr["column"],
                                       self.templates, self._attr["pivot"],
                                       self._attr["filters"], zip_constraints)

    def savefig(self, *args, **kwargs):
        """Save the plot as a PNG fill. This calls into
        :mod:`matplotlib.figure.savefig`
        """

        if self._fig is None:
            self.view()
        self._fig.savefig(*args, **kwargs)

    @abstractmethod
    def set_defaults(self):
        """Sets the default attrs"""
        self._attr["width"] = AttrConf.WIDTH
        self._attr["length"] = AttrConf.LENGTH
        self._attr["per_line"] = AttrConf.PER_LINE
        self._attr["concat"] = AttrConf.CONCAT
        self._attr["filters"] = {}
        self._attr["style"] = True
        self._attr["permute"] = False
        self._attr["pivot"] = AttrConf.PIVOT
        self._attr["xlim"] = AttrConf.XLIM
        self._attr["ylim"] = AttrConf.XLIM
        self._attr["title"] = AttrConf.TITLE
        self._attr["args_to_forward"] = {}
        self._attr["map_label"] = {}
        self._attr["_legend_handles"] = []
        self._attr["_legend_labels"] = []
        self._attr["legend_ncol"] = AttrConf.LEGEND_NCOL

    def view(self, test=False):
        """Displays the graph"""

        if test:
            self._attr["style"] = True
            AttrConf.MPL_STYLE["interactive"] = False

        permute = self._attr["permute"] and not self._attr["concat"]
        if self._attr["style"]:
            with plt.rc_context(AttrConf.MPL_STYLE):
                self._resolve(permute, self._attr["concat"])
        else:
            self._resolve(permute, self._attr["concat"])

    def make_title(self, constraint, pivot, permute, concat):
        """Generates a title string for an axis"""
        if concat:
            return str(constraint)

        if permute:
            return constraint.column
        elif pivot != AttrConf.PIVOT_VAL:
            return "{0}: {1}".format(self._attr["pivot"], self._attr["map_label"].get(pivot, pivot))
        else:
            return ""

    def add_to_legend(self, series_index, handle, constraint, pivot, concat, permute):
        """
        Add series handles and names to the legend
        A handle is returned from a plot on an axis
        e.g. Line2D from axis.plot()
        """
        self._attr["_legend_handles"][series_index] = handle
        legend_labels = self._attr["_legend_labels"]

        if concat and pivot == AttrConf.PIVOT_VAL:
            legend_labels[series_index] = self._attr["column"]
        elif concat:
            legend_labels[series_index] = "{0}: {1}".format(
                self._attr["pivot"],
                self._attr["map_label"].get(pivot, pivot)
            )
        elif permute:
            legend_labels[series_index] = constraint.get_data_name() + ":" + constraint._template.name
        else:
            legend_labels[series_index] = str(constraint)

    def _resolve(self, permute, concat):
        """Determine what data to plot on which axis"""
        pivot_vals, len_pivots = self.c_mgr.generate_pivots(permute)
        pivot_vals = list(pivot_vals)

        num_of_axes = len(self.c_mgr) if concat else len_pivots

        # Create a 2D Layout
        self._layout = PlotLayout(
            self._attr["per_line"],
            num_of_axes,
            width=self._attr["width"],
            length=self._attr["length"],
            title=self._attr['title'])

        self._fig = self._layout.get_fig()

        #Determine what constraint to plot and the corresponding pivot value
        if permute:
            legend_len = self.c_mgr._max_len
            pivots = [y for _, y in pivot_vals]
            c_dict = {c : str(c) for c in self.c_mgr}
            c_list = sorted(c_dict.items(), key=lambda x: (x[1].split(":")[-1], x[1].split(":")[0]))
            constraints = [c[0] for c in c_list]
            cp_pairs = [(c, p) for c in constraints for p in sorted(set(pivots))]
        else:
            legend_len = len_pivots if concat else len(self.c_mgr)
            pivots = pivot_vals
            cp_pairs = [(c, p) for c in self.c_mgr for p in pivots if p in c.result]

        #Initialise legend data and colormap
        self._attr["_legend_handles"] = [None] * legend_len
        self._attr["_legend_labels"] = [None] * legend_len
        self._cmap = ColorMap(legend_len)

        #Group constraints/series with the axis they are to be plotted on
        figure_data = ddict(list)
        for i, (constraint, pivot) in enumerate(cp_pairs):
            axis = self._layout.get_axis(constraint.trace_index if concat else i)
            figure_data[axis].append((constraint, pivot))

        #Plot each axis
        for axis, series_list in figure_data.iteritems():
            self.plot_axis(
                axis,
                series_list,
                permute,
                self._attr["concat"],
                self._attr["args_to_forward"]
            )

        self._fig.legend(self._attr["_legend_handles"],
                         self._attr["_legend_labels"],
                         loc='lower center',
                         ncol=self._attr["legend_ncol"],
                         borderaxespad=0.)

        self._layout.finish(num_of_axes)

    def plot_axis(self, axis, series_list, permute, concat, args_to_forward):
        """Internal Method called to plot data (series_list) on a given axis"""
        raise NotImplementedError("Method Not Implemented")
<commit_msg>plotter: StaticPlot: fix legend and title to show correct labels<commit_after>#    Copyright 2016-2016 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Base matplotlib plotter module"""
from abc import abstractmethod, ABCMeta
from collections import defaultdict as ddict
import matplotlib.pyplot as plt
from trappy.plotter import AttrConf
from trappy.plotter.Constraint import ConstraintManager
from trappy.plotter.PlotLayout import PlotLayout
from trappy.plotter.AbstractDataPlotter import AbstractDataPlotter
from trappy.plotter.ColorMap import ColorMap

class StaticPlot(AbstractDataPlotter):
    """
    This class uses :mod:`trappy.plotter.Constraint.Constraint` to
    represent different permutations of input parameters. These
    constraints are generated by creating an instance of
    :mod:`trappy.plotter.Constraint.ConstraintManager`.

    :param trace: The input data
    :type trace: :mod:`trappy.trace.FTrace` or :mod:`pandas.DataFrame`, list or single

    :param column: specifies the name of the column to
           be plotted.
    :type column: (str, list(str))

    :param templates: TRAPpy events

        .. note::

                This is not required if a :mod:`pandas.DataFrame` is
                used

    :type templates: :mod:`trappy.base.Base`

    :param filters: Filter the column to be plotted as per the
        specified criteria. For Example:
        ::

            filters =
                    {
                        "pid": [ 3338 ],
                        "cpu": [0, 2, 4],
                    }
    :type filters: dict

    :param per_line: Used to control the number of graphs
        in each graph subplot row
    :type per_line: int

    :param concat: Draw all the pivots on a single graph
    :type concat: bool

    :param permute: Draw one plot for each of the traces specified
    :type permute: bool

    :param drawstyle: This argument is forwarded to the matplotlib
        corresponding :func:`matplotlib.pyplot.plot` call

        drawing style.

        .. note::

            step plots are not currently supported for filled
            graphs

    :param xlim: A tuple representing the upper and lower xlimits
    :type xlim: tuple

    :param ylim: A tuple representing the upper and lower ylimits
    :type ylim: tuple

    :param title: A title describing all the generated plots
    :type title: str

    :param style: Created pre-styled graphs loaded from
        :mod:`trappy.plotter.AttrConf.MPL_STYLE`
    :type style: bool

    :param signals: A string of the type event_name:column
        to indicate the value that needs to be plotted

        .. note::

            - Only one of `signals` or both `templates` and
              `columns` should be specified
            - Signals format won't work for :mod:`pandas.DataFrame`
              input

    :type signals: str

    :param legend_ncol: A positive integer that represents the
        number of columns in the legend
    :type legend_ncol: int
    """
    __metaclass__ = ABCMeta

    def __init__(self, traces, templates, **kwargs):
        self._fig = None
        self._layout = None
        super(StaticPlot, self).__init__(traces=traces,
                                         templates=templates)

        self.set_defaults()

        for key in kwargs:
            if key in AttrConf.ARGS_TO_FORWARD:
                self._attr["args_to_forward"][key] = kwargs[key]
            else:
                self._attr[key] = kwargs[key]

        if "signals" in self._attr:
            self._describe_signals()

        self._check_data()

        if "column" not in self._attr:
            raise RuntimeError("Value Column not specified")

        zip_constraints = not self._attr["permute"]
        self.c_mgr = ConstraintManager(traces, self._attr["column"],
                                       self.templates, self._attr["pivot"],
                                       self._attr["filters"], zip_constraints)

    def savefig(self, *args, **kwargs):
        """Save the plot as a PNG fill. This calls into
        :mod:`matplotlib.figure.savefig`
        """

        if self._fig is None:
            self.view()
        self._fig.savefig(*args, **kwargs)

    @abstractmethod
    def set_defaults(self):
        """Sets the default attrs"""
        self._attr["width"] = AttrConf.WIDTH
        self._attr["length"] = AttrConf.LENGTH
        self._attr["per_line"] = AttrConf.PER_LINE
        self._attr["concat"] = AttrConf.CONCAT
        self._attr["filters"] = {}
        self._attr["style"] = True
        self._attr["permute"] = False
        self._attr["pivot"] = AttrConf.PIVOT
        self._attr["xlim"] = AttrConf.XLIM
        self._attr["ylim"] = AttrConf.XLIM
        self._attr["title"] = AttrConf.TITLE
        self._attr["args_to_forward"] = {}
        self._attr["map_label"] = {}
        self._attr["_legend_handles"] = []
        self._attr["_legend_labels"] = []
        self._attr["legend_ncol"] = AttrConf.LEGEND_NCOL

    def view(self, test=False):
        """Displays the graph"""

        if test:
            self._attr["style"] = True
            AttrConf.MPL_STYLE["interactive"] = False

        permute = self._attr["permute"] and not self._attr["concat"]
        if self._attr["style"]:
            with plt.rc_context(AttrConf.MPL_STYLE):
                self._resolve(permute, self._attr["concat"])
        else:
            self._resolve(permute, self._attr["concat"])

    def make_title(self, constraint, pivot, permute, concat):
        """Generates a title string for an axis"""
        if concat:
            return str(constraint)

        if permute:
            return constraint.get_data_name()
        elif pivot != AttrConf.PIVOT_VAL:
            return "{0}: {1}".format(self._attr["pivot"], self._attr["map_label"].get(pivot, pivot))
        else:
            return ""

    def add_to_legend(self, series_index, handle, constraint, pivot, concat, permute):
        """
        Add series handles and names to the legend
        A handle is returned from a plot on an axis
        e.g. Line2D from axis.plot()
        """
        self._attr["_legend_handles"][series_index] = handle
        legend_labels = self._attr["_legend_labels"]

        if concat and pivot == AttrConf.PIVOT_VAL:
            legend_labels[series_index] = self._attr["column"]
        elif concat:
            legend_labels[series_index] = "{0}: {1}".format(
                self._attr["pivot"],
                self._attr["map_label"].get(pivot, pivot)
            )
        elif permute:
            legend_labels[series_index] = constraint._template.name + ":" + constraint.column
        else:
            legend_labels[series_index] = str(constraint)

    def _resolve(self, permute, concat):
        """Determine what data to plot on which axis"""
        pivot_vals, len_pivots = self.c_mgr.generate_pivots(permute)
        pivot_vals = list(pivot_vals)

        num_of_axes = len(self.c_mgr) if concat else len_pivots

        # Create a 2D Layout
        self._layout = PlotLayout(
            self._attr["per_line"],
            num_of_axes,
            width=self._attr["width"],
            length=self._attr["length"],
            title=self._attr['title'])

        self._fig = self._layout.get_fig()

        #Determine what constraint to plot and the corresponding pivot value
        if permute:
            legend_len = self.c_mgr._max_len
            pivots = [y for _, y in pivot_vals]
            c_dict = {c : str(c) for c in self.c_mgr}
            c_list = sorted(c_dict.items(), key=lambda x: (x[1].split(":")[-1], x[1].split(":")[0]))
            constraints = [c[0] for c in c_list]
            cp_pairs = [(c, p) for c in constraints for p in sorted(set(pivots))]
        else:
            legend_len = len_pivots if concat else len(self.c_mgr)
            pivots = pivot_vals
            cp_pairs = [(c, p) for c in self.c_mgr for p in pivots if p in c.result]

        #Initialise legend data and colormap
        self._attr["_legend_handles"] = [None] * legend_len
        self._attr["_legend_labels"] = [None] * legend_len
        self._cmap = ColorMap(legend_len)

        #Group constraints/series with the axis they are to be plotted on
        figure_data = ddict(list)
        for i, (constraint, pivot) in enumerate(cp_pairs):
            axis = self._layout.get_axis(constraint.trace_index if concat else i)
            figure_data[axis].append((constraint, pivot))

        #Plot each axis
        for axis, series_list in figure_data.iteritems():
            self.plot_axis(
                axis,
                series_list,
                permute,
                self._attr["concat"],
                self._attr["args_to_forward"]
            )

        self._fig.legend(self._attr["_legend_handles"],
                         self._attr["_legend_labels"],
                         loc='lower center',
                         ncol=self._attr["legend_ncol"],
                         borderaxespad=0.)

        self._layout.finish(num_of_axes)

    def plot_axis(self, axis, series_list, permute, concat, args_to_forward):
        """Internal Method called to plot data (series_list) on a given axis"""
        raise NotImplementedError("Method Not Implemented")
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# CREATED:2015-03-06 14:24:58 by Brian McFee <brian.mcfee@nyu.edu>
'''Unit tests for JAMS core objects'''

import os
import tempfile
import json
import jsonschema
import six
import warnings
import numpy as np
import pandas as pd

from nose.tools import raises, eq_

import jams

# JObject

def test_jobject_dict():

    data = dict(key1='value 1', key2='value 2')

    J = jams.JObject(**data)

    jdict = J.__dict__

    eq_(data, jdict)

def test_jobject_serialize():

    data = dict(key1='value 1', key2='value 2')

    json_data = json.dumps(data, indent=2)

    J = jams.JObject(**data)

    json_jobject = J.dumps(indent=2)

    eq_(json_data, json_jobject)

def test_jobject_deserialize():

    data = dict(key1='value 1', key2='value 2')

    json_data = json.dumps(data, indent=2)

    J = jams.JObject.loads(json_data)

    json_jobject = J.dumps(indent=2)

    eq_(json_data, json_jobject)

def test_jobject_eq():

    def __test(d1, d2, match):

        J1 = jams.JObject(**d1)
        J2 = jams.JObject(**d2)

        # Test self-equivalence
        assert J1 == J1
        assert J2 == J2

        # Test equivalence in both directions
        assert (J1 == J2) == match
        assert (J2 == J1) == match

    data_1 = dict(key1='value 1', key2='value 2')
    data_2 = dict(key1='value 1', key2='value 2')
    data_3 = dict(key1='value 1', key2='value 3')

    yield __test, data_1, data_1, True
    yield __test, data_1, data_2, True
    yield __test, data_1, data_3, False

def test_jobject_nonzero():

    def __test(d, value):
        J = jams.JObject(**d)

        eq_(J.__nonzero__(), value)

    yield __test, {'key': True}, True
    yield __test, {'key': False}, False
    yield __test, {}, False

# Sandbox

def test_sandbox():

    data = dict(key1='value 1', key2='value 2')

    J = jams.Sandbox(**data)

    for key, value in six.iteritems(data):
        eq_(value, J[key])

# JamsFrame

def test_jamsframe_fields():

    eq_(jams.JamsFrame.fields(), ['time', 'duration', 'value', 'confidence'])

def test_jamsframe_from_df():

    df = pd.DataFrame(data=[[0.0, 1.0, 'a', 0.0],
                            [1.0, 2.0, 'b', 0.0]],
                      columns=['time', 'duration', 'value', 'confidence'])

    jf = jams.JamsFrame.from_dataframe(df)

    # 1. type check
    assert isinstance(jf, jams.JamsFrame)

    # 2. check field order
    eq_(list(jf.keys().values),
        jams.JamsFrame.fields())

    # 3. check field types
    assert jf['time'].dtype == np.dtype('<m8[ns]')
    assert jf['duration'].dtype == np.dtype('<m8[ns]')

    # 4. Check the values
    eq_(list(jf['time']),
        list(pd.to_timedelta([0.0, 1.0], unit='s')))
    eq_(list(jf['duration']),
        list(pd.to_timedelta([1.0, 2.0], unit='s')))
    eq_(list(jf['value']), ['a', 'b'])
    eq_(list(jf['confidence']), [0.0, 0.0])

def test_jamsframe_add_observation():
    df = pd.DataFrame(data=[[0.0, 1.0, 'a', 0.0],
                            [1.0, 2.0, 'b', 0.0]],
                      columns=['time', 'duration', 'value', 'confidence'])

    jf = jams.JamsFrame.from_dataframe(df)

    jf.add_observation(time=3.0, duration=1.0, value='c', confidence=0.0)

    eq_(list(jf['time']),
        list(pd.to_timedelta([0.0, 1.0, 3.0], unit='s')))
    eq_(list(jf['duration']),
        list(pd.to_timedelta([1.0, 2.0, 1.0], unit='s')))
    eq_(list(jf['value']), ['a', 'b', 'c'])
    eq_(list(jf['confidence']), [0.0, 0.0, 0.0])

def test_jamsframe_interval_values():

    df = pd.DataFrame(data=[[0.0, 1.0, 'a', 0.0],
                            [1.0, 2.0, 'b', 0.0]],
                      columns=['time', 'duration', 'value', 'confidence'])

    jf = jams.JamsFrame.from_dataframe(df)

    intervals, values = jf.to_interval_values()

    assert np.allclose(intervals, np.array([[0.0, 1.0], [1.0, 3.0]]))
    eq_(values, ['a', 'b'])

def test_jamsframe_serialize():

    def __test(dense):
        df = pd.DataFrame(data=[[0.0, 1.0, 'a', 0.0],
                                [1.0, 2.0, 'b', 0.0]],
                          columns=['time', 'duration', 'value', 'confidence'])

        jf = jams.JamsFrame.from_dataframe(df)
        jf.dense = dense

        jf_s = jf.__json__

        jf2 = jams.JamsFrame.from_dict(jf_s)

        for key in jams.JamsFrame.fields():
            eq_(list(jf[key]), list(jf2[key]))

    for dense in [False, True]:
        yield __test, dense

# Curator

# AnnotationMetadata

# Annotation

# FileMetadata

# AnnotationArray

# JAMS

def test_load_fail():
    # 1. test bad file path
    # 2. test non-json file

    def __test(filename):
        jams.load(filename)

    # Make a non-existent file
    tdir = tempfile.mkdtemp()
    yield raises(IOError)(__test), os.path.join(tdir, 'nonexistent.jams')
    os.rmdir(tdir)

    # Make a non-json file
    tdir = tempfile.mkdtemp()
    badfile = os.path.join(tdir, 'nonexistent.jams')
    with open(badfile, mode='w') as fp:
        fp.write('some garbage')
    yield raises(ValueError)(__test), os.path.join(tdir, 'nonexistent.jams')
    os.unlink(badfile)
    os.rmdir(tdir)

def test_load_valid():

    # 3. test good jams file with strict validation
    # 4. test good jams file without strict validation

    def __test(filename, valid, strict):
        jams.load(filename, validate=valid, strict=strict)

    fn = 'fixtures/valid.jams'

    for validate in [False, True]:
        for strict in [False, True]:
            yield __test, fn, validate, strict

def test_load_invalid():

    def __test(filename, valid, strict):
        jams.load(filename, validate=valid, strict=strict)

    def __test_warn(filename, valid, strict):
        warnings.resetwarnings()
        with warnings.catch_warnings(record=True) as out:
            jams.load(filename, validate=valid, strict=strict)

        assert len(out) > 0
        assert out[0].category is UserWarning
        assert 'failed validating' in str(out[0].message).lower()

    # 5. test bad jams file with strict validation
    # 6. test bad jams file without strict validation
    fn = 'fixtures/invalid.jams'

    # Test once with no validation
    yield __test, fn, False, False

    # With validation, failure can either be a warning or an exception
    yield raises(jsonschema.ValidationError)(__test), fn, True, True

    yield __test_warn, fn, True, False

<commit_msg>added curator tests<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# CREATED:2015-03-06 14:24:58 by Brian McFee <brian.mcfee@nyu.edu>
'''Unit tests for JAMS core objects'''

import os
import tempfile
import json
import jsonschema
import six
import warnings
import numpy as np
import pandas as pd

from nose.tools import raises, eq_

import jams

# JObject

def test_jobject_dict():

    data = dict(key1='value 1', key2='value 2')

    J = jams.JObject(**data)

    jdict = J.__dict__

    eq_(data, jdict)

def test_jobject_serialize():

    data = dict(key1='value 1', key2='value 2')

    json_data = json.dumps(data, indent=2)

    J = jams.JObject(**data)

    json_jobject = J.dumps(indent=2)

    eq_(json_data, json_jobject)

def test_jobject_deserialize():

    data = dict(key1='value 1', key2='value 2')

    json_data = json.dumps(data, indent=2)

    J = jams.JObject.loads(json_data)

    json_jobject = J.dumps(indent=2)

    eq_(json_data, json_jobject)

def test_jobject_eq():

    def __test(d1, d2, match):

        J1 = jams.JObject(**d1)
        J2 = jams.JObject(**d2)

        # Test self-equivalence
        assert J1 == J1
        assert J2 == J2

        # Test equivalence in both directions
        assert (J1 == J2) == match
        assert (J2 == J1) == match

    data_1 = dict(key1='value 1', key2='value 2')
    data_2 = dict(key1='value 1', key2='value 2')
    data_3 = dict(key1='value 1', key2='value 3')

    yield __test, data_1, data_1, True
    yield __test, data_1, data_2, True
    yield __test, data_1, data_3, False

def test_jobject_nonzero():

    def __test(d, value):
        J = jams.JObject(**d)

        eq_(J.__nonzero__(), value)

    yield __test, {'key': True}, True
    yield __test, {'key': False}, False
    yield __test, {}, False

# Sandbox

def test_sandbox():

    data = dict(key1='value 1', key2='value 2')

    J = jams.Sandbox(**data)

    for key, value in six.iteritems(data):
        eq_(value, J[key])

# JamsFrame

def test_jamsframe_fields():

    eq_(jams.JamsFrame.fields(), ['time', 'duration', 'value', 'confidence'])

def test_jamsframe_from_df():

    df = pd.DataFrame(data=[[0.0, 1.0, 'a', 0.0],
                            [1.0, 2.0, 'b', 0.0]],
                      columns=['time', 'duration', 'value', 'confidence'])

    jf = jams.JamsFrame.from_dataframe(df)

    # 1. type check
    assert isinstance(jf, jams.JamsFrame)

    # 2. check field order
    eq_(list(jf.keys().values),
        jams.JamsFrame.fields())

    # 3. check field types
    assert jf['time'].dtype == np.dtype('<m8[ns]')
    assert jf['duration'].dtype == np.dtype('<m8[ns]')

    # 4. Check the values
    eq_(list(jf['time']),
        list(pd.to_timedelta([0.0, 1.0], unit='s')))
    eq_(list(jf['duration']),
        list(pd.to_timedelta([1.0, 2.0], unit='s')))
    eq_(list(jf['value']), ['a', 'b'])
    eq_(list(jf['confidence']), [0.0, 0.0])

def test_jamsframe_add_observation():
    df = pd.DataFrame(data=[[0.0, 1.0, 'a', 0.0],
                            [1.0, 2.0, 'b', 0.0]],
                      columns=['time', 'duration', 'value', 'confidence'])

    jf = jams.JamsFrame.from_dataframe(df)

    jf.add_observation(time=3.0, duration=1.0, value='c', confidence=0.0)

    eq_(list(jf['time']),
        list(pd.to_timedelta([0.0, 1.0, 3.0], unit='s')))
    eq_(list(jf['duration']),
        list(pd.to_timedelta([1.0, 2.0, 1.0], unit='s')))
    eq_(list(jf['value']), ['a', 'b', 'c'])
    eq_(list(jf['confidence']), [0.0, 0.0, 0.0])

def test_jamsframe_interval_values():

    df = pd.DataFrame(data=[[0.0, 1.0, 'a', 0.0],
                            [1.0, 2.0, 'b', 0.0]],
                      columns=['time', 'duration', 'value', 'confidence'])

    jf = jams.JamsFrame.from_dataframe(df)

    intervals, values = jf.to_interval_values()

    assert np.allclose(intervals, np.array([[0.0, 1.0], [1.0, 3.0]]))
    eq_(values, ['a', 'b'])

def test_jamsframe_serialize():

    def __test(dense):
        df = pd.DataFrame(data=[[0.0, 1.0, 'a', 0.0],
                                [1.0, 2.0, 'b', 0.0]],
                          columns=['time', 'duration', 'value', 'confidence'])

        jf = jams.JamsFrame.from_dataframe(df)
        jf.dense = dense

        jf_s = jf.__json__

        jf2 = jams.JamsFrame.from_dict(jf_s)

        for key in jams.JamsFrame.fields():
            eq_(list(jf[key]), list(jf2[key]))

    for dense in [False, True]:
        yield __test, dense

# Curator
def test_curator():

    c = jams.Curator(name='myself', email='you@me.com')

    eq_(c.name, 'myself')
    eq_(c.email, 'you@me.com')

# AnnotationMetadata

# Annotation

# FileMetadata

# AnnotationArray

# JAMS

def test_load_fail():
    # 1. test bad file path
    # 2. test non-json file

    def __test(filename):
        jams.load(filename)

    # Make a non-existent file
    tdir = tempfile.mkdtemp()
    yield raises(IOError)(__test), os.path.join(tdir, 'nonexistent.jams')
    os.rmdir(tdir)

    # Make a non-json file
    tdir = tempfile.mkdtemp()
    badfile = os.path.join(tdir, 'nonexistent.jams')
    with open(badfile, mode='w') as fp:
        fp.write('some garbage')
    yield raises(ValueError)(__test), os.path.join(tdir, 'nonexistent.jams')
    os.unlink(badfile)
    os.rmdir(tdir)

def test_load_valid():

    # 3. test good jams file with strict validation
    # 4. test good jams file without strict validation

    def __test(filename, valid, strict):
        jams.load(filename, validate=valid, strict=strict)

    fn = 'fixtures/valid.jams'

    for validate in [False, True]:
        for strict in [False, True]:
            yield __test, fn, validate, strict

def test_load_invalid():

    def __test(filename, valid, strict):
        jams.load(filename, validate=valid, strict=strict)

    def __test_warn(filename, valid, strict):
        warnings.resetwarnings()
        with warnings.catch_warnings(record=True) as out:
            jams.load(filename, validate=valid, strict=strict)

        assert len(out) > 0
        assert out[0].category is UserWarning
        assert 'failed validating' in str(out[0].message).lower()

    # 5. test bad jams file with strict validation
    # 6. test bad jams file without strict validation
    fn = 'fixtures/invalid.jams'

    # Test once with no validation
    yield __test, fn, False, False

    # With validation, failure can either be a warning or an exception
    yield raises(jsonschema.ValidationError)(__test), fn, True, True

    yield __test_warn, fn, True, False

<|endoftext|>


<commit_before>#checks for updates, then runs the actual tracker
import json
import os, shutil
import threading
import urllib2
import logging
import webbrowser
import zipfile
from StringIO import StringIO
from Tkinter import *
import errno
import traceback

import time

import subprocess
from enum import Enum

class UpdateStep(Enum):
    PROMPTING = "Asking about update"
    PRE_DOWNLOAD = "Preparing for update"
    DOWNLOAD = "Downloading update zip"
    EXTRACT = "Extracting update zip"
    PERFORMING = "Performing update"
    DONE = "Update finished"
    ERROR = "Error"
    SKIP = "Skipping update"

# specific kinds of errors where we can actionably tell the user what to do to fix it, not just "please report"
class UpdateError(Enum):
    TRACKER_RUNNING = 1

wdir_prefix = "../"
update_option_name = "check_for_updates"

error_log = logging.getLogger("tracker")
error_log.addHandler(logging.FileHandler(wdir_prefix + "tracker_log.txt", mode='a'))
error_log.setLevel(logging.INFO)

def log_error(msg):
    # Print it to stdout for dev troubleshooting, log it to a file for production
    msg = time.strftime("%Y-%m-%d %H:%M:%S ") + msg
    print(msg)
    error_log.error(msg)

# got this from some guy on stackoverflow
def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc:  # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

# also got this from some guy on stackoverflow.
# I just want a version of shutil.copytree that can overwrite into an existing dest dir. that's what this is.
def recursive_overwrite(src, dest, ignore=None):
    if os.path.isdir(src):
        if not os.path.isdir(dest):
            os.makedirs(dest)
        files = os.listdir(src)
        if ignore is not None:
            ignored = ignore(src, files)
        else:
            ignored = set()
        for f in files:
            if f not in ignored:
                recursive_overwrite(os.path.join(src, f),
                                    os.path.join(dest, f),
                                    ignore)
    else:
        shutil.copyfile(src, dest)

class Updater(object):
    def __init__(self):
        self.latest_version = ""

        self.root = None
        self.update_thread = None
        self.update_step = None
        self.update_error = None

        self.options_file = wdir_prefix + "options.json"
        if not os.path.isfile(self.options_file):
            shutil.copy(wdir_prefix + "tracker-lib/options_default.json", self.options_file)

        with open(self.options_file, "r") as options_json:
            self.options = json.load(options_json)

        with open(wdir_prefix + 'version.txt', 'r') as f:
            self.current_version = f.read()

    # creates tk window and blocks until it goes away
    def check_if_update_possible(self):
        try:
            #if the update option isn't present, the default is to do updates.
            if update_option_name not in self.options or self.options[update_option_name]:
                # check if github has a newer version than us
                latest = "https://api.github.com/repos/Hyphen-ated/RebirthItemTrackerTest/releases/latest"
                github_info_json = urllib2.urlopen(latest).read()
                info = json.loads(github_info_json)
                self.latest_version = info["name"]
                if self.latest_version != self.current_version:
                    return True
        except Exception:
            log_error("Error while checking whether there's a new version:\n" + traceback.format_exc())
        return False

    def create_update_window(self):
        self.update_step = UpdateStep.PROMPTING
        self.root = Tk()
        self.root.wm_title("Update Item Tracker")
        self.root.resizable(False, False)
        self.root.minsize(500, 300)
        self.label = Label(self.root, text="Your current version is " + self.current_version + "\nThe latest version is " + self.latest_version)
        self.label.pack()

        self.update = Button(self.root, text="Update Now", command=self.trigger_update_thread)
        self.update.pack()

        self.justlaunchit = Button(self.root, text="Ignore Update This Time", command=self.just_launch_it)
        self.justlaunchit.pack()

        self.ignore = Button(self.root, text="Ignore Updates Forever", command=self.ignore_updates)
        self.ignore.pack()
        mainloop()

    def trigger_update_thread(self):
        self.update_step = UpdateStep.PRE_DOWNLOAD
        self.update.pack_forget()
        self.justlaunchit.pack_forget()
        self.ignore.pack_forget()

        self.update_thread = threading.Thread(target=self.do_update)
        self.update_thread.start()
        self.check_update_status()

    def check_update_status(self):
        self.label['text'] = "Updating, please wait...\n" + self.update_step.value
        if self.update_step == UpdateStep.DONE:
            self.root.destroy()
            return

        if self.update_step == UpdateStep.ERROR:
            self.handle_error()
        else:
            self.root.after(200, self.check_update_status)

    def handle_error(self):
        reportbtn = Button(self.root, text="Open bug report page", command=self.open_report_page)
        ignorebtn = Button(self.root, text="Ignore Updates Forever", command=self.ignore_updates)
        closebtn = Button(self.root, text="OK", command=self.dont_launch)

        if self.update_error == UpdateError.TRACKER_RUNNING:
            self.label['text']="Another copy of the tracker seems to be running already, close it and retry.\n"+ \
                               "If that doesn't work, please report this bug and include your tracker_log.txt"
            reportbtn.pack()
            closebtn.pack()
        else:
            # generic problem we don't have special handling for
            self.label['text'] = "Sorry, there was an error during the update.\n"+ \
                                 "You might need to just manually download the new version\n"+ \
                                 "and copy in your old options.json (if you care about your settings.)\n"+ \
                                 "Please report this bug and include your tracker_log.txt, as well as what version you had."
            reportbtn.pack()
            ignorebtn.pack()
            closebtn.pack()

    def open_report_page(self):
        webbrowser.open("https://github.com/Hyphen-ated/RebirthItemTracker/issues", autoraise=True)
        self.root.destroy()

    def do_update(self):
        try:
            tasklist = subprocess.check_output('tasklist', shell=True)
            if "item_tracker.exe" in tasklist:
                self.update_step = UpdateStep.ERROR
                self.update_error = UpdateError.TRACKER_RUNNING
                return

            backupdir = wdir_prefix + "options backups/" + self.current_version
            mkdir_p(backupdir)
            shutil.copy(wdir_prefix + "options.json", backupdir)

            scratch = wdir_prefix + "update_scratchdir/"
            if os.path.exists(scratch):
                shutil.rmtree(scratch)

            mkdir_p(scratch)
            try:
                self.update_step = UpdateStep.DOWNLOAD
                url = 'https://github.com/Hyphen-ated/RebirthItemTrackerTest/releases/download/' + self.latest_version + '/Rebirth.Item.Tracker-' + self.latest_version + ".zip"
                urlstream = urllib2.urlopen(url)
                myzip = zipfile.ZipFile(StringIO(urlstream.read()))
                self.update_step = UpdateStep.EXTRACT
                myzip.extractall(scratch)
            except Exception as e:
                log_error("Failed to download and extract latest version from GitHub ( url was :" + url + " )\n" + traceback.format_exc())
                self.update_step = UpdateStep.ERROR
                return

            self.update_step = UpdateStep.PERFORMING
            innerdir = scratch + "Rebirth Item Tracker/"

            with open(wdir_prefix + "tracker-lib/options_default.json", "r") as old_defaults_json:
                old_defaults = json.load(old_defaults_json)

            with open(innerdir + "tracker-lib/options_default.json", "r") as new_defaults_json:
                new_defaults = json.load(new_defaults_json)

            for k,v in old_defaults.iteritems():
                # for each default option they left unchanged, if the default changed in the new version, give them the new default
                if k in self.options and self.options[k] == v and new_defaults[k] != v:
                    self.options[k] = new_defaults[k]

            self.write_options()

            path_to_exe = wdir_prefix + "tracker-lib/item_tracker.exe"
            # the exe might not exist if we're trying to recover from a previous botched update, that's okay
            if os.path.exists(path_to_exe):
                try:
                   os.remove(path_to_exe)
                except Exception:
                    # failing to remove the exe means very likely it's currently running
                    self.update_step = UpdateStep.ERROR
                    self.update_error = UpdateError.TRACKER_RUNNING
                    log_error(traceback.format_exc())
                    return

            for subdir in ["collectibles", "overlay text", "tracker-lib"]:
                dir_to_remove = wdir_prefix + subdir
                if os.path.isdir(dir_to_remove):
                    shutil.rmtree(dir_to_remove)

            shutil.move(innerdir + "updater-lib", scratch)
            recursive_overwrite(innerdir, "..")
            self.update_step = UpdateStep.DONE
        except Exception:
            log_error("Error while attempting tracker update\n" + traceback.format_exc())
            self.update_step = UpdateStep.ERROR

    def just_launch_it(self):
        self.update_step = UpdateStep.SKIP
        self.root.destroy()

    def ignore_updates(self):
        self.options[update_option_name] = False
        self.write_options()
        self.update_step = UpdateStep.SKIP
        self.root.destroy()

    def dont_launch(self):
        self.update_step = UpdateStep.ERROR
        self.root.destroy()

    def write_options(self):
        with open(self.options_file, "w") as json_file:
            json.dump(self.options, json_file, indent=3, sort_keys=True)

def main():
    try:
        updater = Updater()
        if updater.check_if_update_possible():
            #blocks until either an update is finished or they skip the update
            updater.create_update_window()

        if updater.update_step == UpdateStep.ERROR:
            return

        print("launching tracker")
        os.chdir(wdir_prefix + "tracker-lib/")
        os.execl("item_tracker.exe", "Rebirth Item Tracker")
    except Exception:
        log_error("Error with tracker updater outside of the actual update process\n" + traceback.format_exc())

main()

<commit_msg>point from trackertest back to the real repo<commit_after>#checks for updates, then runs the actual tracker
import json
import os, shutil
import threading
import urllib2
import logging
import webbrowser
import zipfile
from StringIO import StringIO
from Tkinter import *
import errno
import traceback

import time

import subprocess
from enum import Enum

class UpdateStep(Enum):
    PROMPTING = "Asking about update"
    PRE_DOWNLOAD = "Preparing for update"
    DOWNLOAD = "Downloading update zip"
    EXTRACT = "Extracting update zip"
    PERFORMING = "Performing update"
    DONE = "Update finished"
    ERROR = "Error"
    SKIP = "Skipping update"

# specific kinds of errors where we can actionably tell the user what to do to fix it, not just "please report"
class UpdateError(Enum):
    TRACKER_RUNNING = 1

wdir_prefix = "../"
update_option_name = "check_for_updates"

error_log = logging.getLogger("tracker")
error_log.addHandler(logging.FileHandler(wdir_prefix + "tracker_log.txt", mode='a'))
error_log.setLevel(logging.INFO)

def log_error(msg):
    # Print it to stdout for dev troubleshooting, log it to a file for production
    msg = time.strftime("%Y-%m-%d %H:%M:%S ") + msg
    print(msg)
    error_log.error(msg)

# got this from some guy on stackoverflow
def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc:  # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

# also got this from some guy on stackoverflow.
# I just want a version of shutil.copytree that can overwrite into an existing dest dir. that's what this is.
def recursive_overwrite(src, dest, ignore=None):
    if os.path.isdir(src):
        if not os.path.isdir(dest):
            os.makedirs(dest)
        files = os.listdir(src)
        if ignore is not None:
            ignored = ignore(src, files)
        else:
            ignored = set()
        for f in files:
            if f not in ignored:
                recursive_overwrite(os.path.join(src, f),
                                    os.path.join(dest, f),
                                    ignore)
    else:
        shutil.copyfile(src, dest)

class Updater(object):
    def __init__(self):
        self.latest_version = ""

        self.root = None
        self.update_thread = None
        self.update_step = None
        self.update_error = None

        self.options_file = wdir_prefix + "options.json"
        if not os.path.isfile(self.options_file):
            shutil.copy(wdir_prefix + "tracker-lib/options_default.json", self.options_file)

        with open(self.options_file, "r") as options_json:
            self.options = json.load(options_json)

        with open(wdir_prefix + 'version.txt', 'r') as f:
            self.current_version = f.read()

    # creates tk window and blocks until it goes away
    def check_if_update_possible(self):
        try:
            #if the update option isn't present, the default is to do updates.
            if update_option_name not in self.options or self.options[update_option_name]:
                # check if github has a newer version than us
                latest = "https://api.github.com/repos/Hyphen-ated/RebirthItemTracker/releases/latest"
                github_info_json = urllib2.urlopen(latest).read()
                info = json.loads(github_info_json)
                self.latest_version = info["name"]
                if self.latest_version != self.current_version:
                    return True
        except Exception:
            log_error("Error while checking whether there's a new version:\n" + traceback.format_exc())
        return False

    def create_update_window(self):
        self.update_step = UpdateStep.PROMPTING
        self.root = Tk()
        self.root.wm_title("Update Item Tracker")
        self.root.resizable(False, False)
        self.root.minsize(500, 300)
        self.label = Label(self.root, text="Your current version is " + self.current_version + "\nThe latest version is " + self.latest_version)
        self.label.pack()

        self.update = Button(self.root, text="Update Now", command=self.trigger_update_thread)
        self.update.pack()

        self.justlaunchit = Button(self.root, text="Ignore Update This Time", command=self.just_launch_it)
        self.justlaunchit.pack()

        self.ignore = Button(self.root, text="Ignore Updates Forever", command=self.ignore_updates)
        self.ignore.pack()
        mainloop()

    def trigger_update_thread(self):
        self.update_step = UpdateStep.PRE_DOWNLOAD
        self.update.pack_forget()
        self.justlaunchit.pack_forget()
        self.ignore.pack_forget()

        self.update_thread = threading.Thread(target=self.do_update)
        self.update_thread.start()
        self.check_update_status()

    def check_update_status(self):
        self.label['text'] = "Updating, please wait...\n" + self.update_step.value
        if self.update_step == UpdateStep.DONE:
            self.root.destroy()
            return

        if self.update_step == UpdateStep.ERROR:
            self.handle_error()
        else:
            self.root.after(200, self.check_update_status)

    def handle_error(self):
        reportbtn = Button(self.root, text="Open bug report page", command=self.open_report_page)
        ignorebtn = Button(self.root, text="Ignore Updates Forever", command=self.ignore_updates)
        closebtn = Button(self.root, text="OK", command=self.dont_launch)

        if self.update_error == UpdateError.TRACKER_RUNNING:
            self.label['text']="Another copy of the tracker seems to be running already, close it and retry.\n"+ \
                               "If that doesn't work, please report this bug and include your tracker_log.txt"
            reportbtn.pack()
            closebtn.pack()
        else:
            # generic problem we don't have special handling for
            self.label['text'] = "Sorry, there was an error during the update.\n"+ \
                                 "You might need to just manually download the new version\n"+ \
                                 "and copy in your old options.json (if you care about your settings.)\n"+ \
                                 "Please report this bug and include your tracker_log.txt, as well as what version you had."
            reportbtn.pack()
            ignorebtn.pack()
            closebtn.pack()

    def open_report_page(self):
        webbrowser.open("https://github.com/Hyphen-ated/RebirthItemTracker/issues", autoraise=True)
        self.root.destroy()

    def do_update(self):
        try:
            tasklist = subprocess.check_output('tasklist', shell=True)
            if "item_tracker.exe" in tasklist:
                self.update_step = UpdateStep.ERROR
                self.update_error = UpdateError.TRACKER_RUNNING
                return

            backupdir = wdir_prefix + "options backups/" + self.current_version
            mkdir_p(backupdir)
            shutil.copy(wdir_prefix + "options.json", backupdir)

            scratch = wdir_prefix + "update_scratchdir/"
            if os.path.exists(scratch):
                shutil.rmtree(scratch)

            mkdir_p(scratch)
            try:
                self.update_step = UpdateStep.DOWNLOAD
                url = 'https://github.com/Hyphen-ated/RebirthItemTracker/releases/download/' + self.latest_version + '/Rebirth.Item.Tracker-' + self.latest_version + ".zip"
                urlstream = urllib2.urlopen(url)
                myzip = zipfile.ZipFile(StringIO(urlstream.read()))
                self.update_step = UpdateStep.EXTRACT
                myzip.extractall(scratch)
            except Exception as e:
                log_error("Failed to download and extract latest version from GitHub ( url was :" + url + " )\n" + traceback.format_exc())
                self.update_step = UpdateStep.ERROR
                return

            self.update_step = UpdateStep.PERFORMING
            innerdir = scratch + "Rebirth Item Tracker/"

            with open(wdir_prefix + "tracker-lib/options_default.json", "r") as old_defaults_json:
                old_defaults = json.load(old_defaults_json)

            with open(innerdir + "tracker-lib/options_default.json", "r") as new_defaults_json:
                new_defaults = json.load(new_defaults_json)

            for k,v in old_defaults.iteritems():
                # for each default option they left unchanged, if the default changed in the new version, give them the new default
                if k in self.options and self.options[k] == v and new_defaults[k] != v:
                    self.options[k] = new_defaults[k]

            self.write_options()

            path_to_exe = wdir_prefix + "tracker-lib/item_tracker.exe"
            # the exe might not exist if we're trying to recover from a previous botched update, that's okay
            if os.path.exists(path_to_exe):
                try:
                   os.remove(path_to_exe)
                except Exception:
                    # failing to remove the exe means very likely it's currently running
                    self.update_step = UpdateStep.ERROR
                    self.update_error = UpdateError.TRACKER_RUNNING
                    log_error(traceback.format_exc())
                    return

            for subdir in ["collectibles", "overlay text", "tracker-lib"]:
                dir_to_remove = wdir_prefix + subdir
                if os.path.isdir(dir_to_remove):
                    shutil.rmtree(dir_to_remove)

            shutil.move(innerdir + "updater-lib", scratch)
            recursive_overwrite(innerdir, "..")
            self.update_step = UpdateStep.DONE
        except Exception:
            log_error("Error while attempting tracker update\n" + traceback.format_exc())
            self.update_step = UpdateStep.ERROR

    def just_launch_it(self):
        self.update_step = UpdateStep.SKIP
        self.root.destroy()

    def ignore_updates(self):
        self.options[update_option_name] = False
        self.write_options()
        self.update_step = UpdateStep.SKIP
        self.root.destroy()

    def dont_launch(self):
        self.update_step = UpdateStep.ERROR
        self.root.destroy()

    def write_options(self):
        with open(self.options_file, "w") as json_file:
            json.dump(self.options, json_file, indent=3, sort_keys=True)

def main():
    try:
        updater = Updater()
        if updater.check_if_update_possible():
            #blocks until either an update is finished or they skip the update
            updater.create_update_window()

        if updater.update_step == UpdateStep.ERROR:
            return

        print("launching tracker")
        os.chdir(wdir_prefix + "tracker-lib/")
        os.execl("item_tracker.exe", "Rebirth Item Tracker")
    except Exception:
        log_error("Error with tracker updater outside of the actual update process\n" + traceback.format_exc())

main()

<|endoftext|>


<commit_before># Copyright (C) 2014 Forest and Biomass Romania
# Copyright (C) 2020 NextERP Romania
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

{
    "name": "Romania - Localization Config",
    "summary": "Romania - Localization Install and Config Applications",
    "license": "AGPL-3",
    "version": "14.0.2.1.0",
    "author": "NextERP Romania,"
    "Forest and Biomass Romania,"
    "Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/l10n-romania",
    "depends": ["l10n_ro", "base_vat"],
    "data": [
        "views/account_journal.xml",
        "views/common_report.xml",
        "views/res_config_view.xml",
        "views/res_bank_view.xml",
        "views/res_partner_view.xml",
    ],
    "pre_init_hook": "pre_init_hook",
    "development_status": "Mature",
    "maintainers": ["feketemihai"],
    "installable": True,
    "auto_install": True,
}
<commit_msg>l10n_ro_config 14.0.2.2.0<commit_after># Copyright (C) 2014 Forest and Biomass Romania
# Copyright (C) 2020 NextERP Romania
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

{
    "name": "Romania - Localization Config",
    "summary": "Romania - Localization Install and Config Applications",
    "license": "AGPL-3",
    "version": "14.0.2.2.0",
    "author": "NextERP Romania,"
    "Forest and Biomass Romania,"
    "Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/l10n-romania",
    "depends": ["l10n_ro", "base_vat"],
    "data": [
        "views/account_journal.xml",
        "views/common_report.xml",
        "views/res_config_view.xml",
        "views/res_bank_view.xml",
        "views/res_partner_view.xml",
    ],
    "pre_init_hook": "pre_init_hook",
    "development_status": "Mature",
    "maintainers": ["feketemihai"],
    "installable": True,
    "auto_install": True,
}
<|endoftext|>