<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

AUTHOR = u'Roger González'
SITENAME = u'Roger González - rogergonzalez.com.ve'
SITETITLE = 'Roger González'
SITEURL = 'http://localhost:8000'
#SITEURL = 'http://rogergonzalez21.github.io'

PATH = 'content'

TIMEZONE = 'America/Caracas'

DEFAULT_LANG = u'en'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

PLUGIN_PATHS = ['./pelican-plugins']
PLUGINS = ['i18n_subsites']

# Language configs
I18N_SUBSITES = {
    'es': {
        'MENUITEMS': (('Acerca de', '/es/pages/about/'),
			 ('Curriculum', '/es/pages/curriculum/'),
			 ('Archivos', '/es/archives/'),
             ('Categorías', '/es/category/'),
             ('Tags', '/es/tag/'),
             ('English', '/')),
       	'THEME': 'svbhack-theme',
       	'TAGLINE': 'Pythonista. De buen comer. Crítico. Amante de la música. Siempre con ganas de aprender.',
        }
    }

# Other configs
DISPLAY_PAGES_ON_MENU = False
DEFAULT_DATE_FORMAT = '%d/%m/%Y'
SUMMARY_MAX_LENGTH = 50
SLUGIFY_SOURCE = 'title'
STATIC_PATHS = ['images']
THEME = 'svbhack-theme'
USER_LOGO_URL = SITEURL + '/images/yo.png'
FAVICON = SITEURL + '/images/favicon.ico'
GOOGLE_ANALYTICS = 'UA-69257856-1'
DISQUS_SITENAME = 'rogergonzalez21'
TAGLINE = 'Pythonista. Love to eat. Critic. Music lover. Always wanting to learn more.'
DELETE_OUTPUT_DIRECTORY = False
DEFAULT_METADATA = {
    'status': 'draft',
}
MAIN_MENU = True
MENUITEMS = (('About', '/pages/about/'),
			 ('Curriculum Vitae', '/pages/curriculum/'),
			 ('Archive', '/archives/'),
             ('Category', '/category/'),
             ('Tags', '/tag/'),
             ('Español', '/es/'))

# URL's
ARTICLE_URL = '{date:%Y}/{date:%m}/{slug}/'
ARTICLE_SAVE_AS = '{date:%Y}/{date:%m}/{slug}/index.html'
DRAFT_URL = 'drafts/{slug}/'
DRAFT_SAVE_AS = 'drafts/{slug}.html'
PAGE_URL = 'pages/{slug}/'
PAGE_SAVE_AS = 'pages/{slug}/index.html'
CATEGORY_URL = 'category/{slug}/'
CATEGORY_SAVE_AS = 'category/{slug}/index.html'
TAG_URL = 'tag/{slug}/'
TAG_SAVE_AS = 'tag/{slug}/index.html'
AUTHOR_URL = 'author/{slug}/'
AUTHOR_SAVE_AS = 'author/{slug}/index.html'
YEAR_ARCHIVE_SAVE_AS = '{date:%Y}/index.html'
TAGS_URL = 'tag/'
TAGS_SAVE_AS = 'tag/index.html'
CATEGORIES_URL = 'category/'
CATEGORIES_SAVE_AS = 'category/index.html'
ARCHIVES_URL = 'archives/'
ARCHIVES_SAVE_AS = 'archives/index.html'

# Social widget
SOCIAL = (('linkedin', 'https://ve.linkedin.com/in/rogergonzalez21'),
          ('github', 'https://github.com/Rogergonzalez21'),
          ('envelope-o', 'mailto:rogergonzalez21@gmail.com'))

DEFAULT_PAGINATION = 5

# Uncomment following line if you want document-relative URLs when developing
#RELATIVE_URLS = True
<commit_msg>Changed the domain to rogerandresgonzalez.com.ve<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

AUTHOR = u'Roger Andrés González'
SITENAME = u'Roger Andrés González - rogerandresgonzalez.com.ve'
SITETITLE = 'Roger Andrés González'
SITEURL = 'http://localhost:8000'
#SITEURL = 'http://rogerandresgonzalez.com.ve'

PATH = 'content'

TIMEZONE = 'America/Caracas'

DEFAULT_LANG = u'en'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

PLUGIN_PATHS = ['./pelican-plugins']
PLUGINS = ['i18n_subsites']

# Language configs
I18N_SUBSITES = {
    'es': {
        'MENUITEMS': (('Acerca de', '/es/pages/about/'),
			 ('Curriculum', '/es/pages/curriculum/'),
			 ('Archivos', '/es/archives/'),
             ('Categorías', '/es/category/'),
             ('Tags', '/es/tag/'),
             ('English', '/')),
       	'THEME': 'svbhack-theme',
       	'TAGLINE': 'Pythonista. De buen comer. Crítico. Amante de la música. Siempre con ganas de aprender.',
        }
    }

# Other configs
DISPLAY_PAGES_ON_MENU = False
DEFAULT_DATE_FORMAT = '%d/%m/%Y'
SUMMARY_MAX_LENGTH = 50
SLUGIFY_SOURCE = 'title'
STATIC_PATHS = ['images']
THEME = 'svbhack-theme'
USER_LOGO_URL = SITEURL + '/images/yo.png'
FAVICON = SITEURL + '/images/favicon.ico'
GOOGLE_ANALYTICS = 'UA-69257856-1'
DISQUS_SITENAME = 'rogergonzalez21'
TAGLINE = 'Pythonista. Love to eat. Critic. Music lover. Always wanting to learn more.'
DELETE_OUTPUT_DIRECTORY = False
DEFAULT_METADATA = {
    'status': 'draft',
}
MAIN_MENU = True
MENUITEMS = (('About', '/pages/about/'),
			 ('Curriculum Vitae', '/pages/curriculum/'),
			 ('Archive', '/archives/'),
             ('Category', '/category/'),
             ('Tags', '/tag/'),
             ('Español', '/es/'))

# URL's
ARTICLE_URL = '{date:%Y}/{date:%m}/{slug}/'
ARTICLE_SAVE_AS = '{date:%Y}/{date:%m}/{slug}/index.html'
DRAFT_URL = 'drafts/{slug}/'
DRAFT_SAVE_AS = 'drafts/{slug}.html'
PAGE_URL = 'pages/{slug}/'
PAGE_SAVE_AS = 'pages/{slug}/index.html'
CATEGORY_URL = 'category/{slug}/'
CATEGORY_SAVE_AS = 'category/{slug}/index.html'
TAG_URL = 'tag/{slug}/'
TAG_SAVE_AS = 'tag/{slug}/index.html'
AUTHOR_URL = 'author/{slug}/'
AUTHOR_SAVE_AS = 'author/{slug}/index.html'
YEAR_ARCHIVE_SAVE_AS = '{date:%Y}/index.html'
TAGS_URL = 'tag/'
TAGS_SAVE_AS = 'tag/index.html'
CATEGORIES_URL = 'category/'
CATEGORIES_SAVE_AS = 'category/index.html'
ARCHIVES_URL = 'archives/'
ARCHIVES_SAVE_AS = 'archives/index.html'

# Social widget
SOCIAL = (('linkedin', 'https://ve.linkedin.com/in/rogergonzalez21'),
          ('github', 'https://github.com/Rogergonzalez21'),
          ('envelope-o', 'mailto:rogergonzalez21@gmail.com'))

DEFAULT_PAGINATION = 5

# Uncomment following line if you want document-relative URLs when developing
#RELATIVE_URLS = True
<|endoftext|>


<commit_before>#!/usr/bin/env python

from subprocess import Popen, PIPE
import time
import threading
import unittest
import os
import urllib2
import codecs
try:
    import json
except:
    import simplejson as json

# Barrister conformance test runner

home = os.environ["HOME"]

#
# Java config
#
m2_jackson = "%s/.m2/repository/org/codehaus/jackson" % home
jackson_deps = [ "jackson-mapper-asl/1.9.4/jackson-mapper-asl-1.9.4.jar",
                 "jackson-core-asl/1.9.4/jackson-core-asl-1.9.4.jar" ]
barrister_java = "../../barrister-java"
if os.environ.has_key("BARRISTER_JAVA"):
    barrister_java = os.environ["BARRISTER_JAVA"]
java_cp = "%s/target/classes:%s/conform/target/classes" % (barrister_java, barrister_java)
for d in jackson_deps:
    java_cp += ":%s/%s" % (m2_jackson, d)
winstone_jar = "%s/conform/lib/winstone-0.9.10.jar" % barrister_java
java_war = "%s/conform/target/barrister-conform-test.war" % barrister_java

#
# Clients to run against each server
#
clients = [
    # format: name, command line
    [ "python-client", ["python", "client.py"] ],
    [ "java-client", ["java", "-cp", java_cp, "com.bitmechanic.barrister.conform.Client" ] ]
]

verbose = os.environ.has_key('CONFORM_VERBOSE')
def log(msg):
    if verbose:
        print msg.strip()

class Runner(threading.Thread):

    def __init__(self, name, cmd):
        super(Runner, self).__init__()
        self.name = name
        self.cmd = cmd
        self.exit_code = None
        self.proc = None

    def run(self):
        log("[%s] Starting process: %s" % (self.name, " ".join(self.cmd)))
        self.proc = Popen(self.cmd, stdout=PIPE, stderr=PIPE, close_fds=False,
                          shell=False)
        out, err = self.proc.communicate()
        for line in out.split("\n"):
            log("[%s out] %s" % (self.name, line))
        for line in err.split("\n"):
            log("[%s err] %s" % (self.name, line))
        self.exit_code = self.proc.poll()

class ConformTest(unittest.TestCase):

    def test_python_server(self):
        cmd = ["python", "flask_server.py", "conform.json"]
        self._test_server(1, "python-flask", cmd)

    def _test_java_server(self):
        cmd = ["java", "-DidlJson=conform.json", "-jar", winstone_jar,
               "--httpPort=9233", java_war ]
        self._test_server(3, "java", cmd)

    def _test_server(self, sleep_time, s_name, s_cmd):
        errs = [ ]
        expected = [ ]
        infile = "conform.in"
        inf = codecs.open(infile, "r", "utf-8")
        for line in inf:
            line = line.strip()
            if line != "" and line.find("#") != 0 and len(line.split("|")) == 5:
                expected.append(line)
        inf.close()
        try:
            s_proc = Runner(s_name, s_cmd)
            s_proc.start()
            time.sleep(sleep_time)
            for c_name, c_cmd in clients:
                print "Testing client '%s' vs server '%s'" % (c_name, s_name)
                outfile = "%s-to-%s.out" % (c_name, s_name)
                c_cmd.extend([infile, outfile])
                c_proc = Runner(c_name, c_cmd)
                c_proc.start()
                c_proc.join()
                if c_proc.exit_code != 0:
                    errs.append("client %s failed - %d" % (c_name, c_proc.exit_code))
                else:
                    out = codecs.open(outfile, "r", "utf-8")
                    i = 0
                    fails = []
                    for line in out:
                        e_cols = expected[i].split("|")
                        l_cols = line.split("|")
                        match = True
                        #print expected[i]
                        #print line
                        if len(e_cols) != len(l_cols):
                            match = False
                        elif e_cols[3] != l_cols[3]:
                            match = False
                        elif json.loads(e_cols[4]) != json.loads(l_cols[4]):
                            match = False
                        if not match:
                            msg = "line %d: %s != %s" % (i, expected[i], line)
                            fails.append(msg)
                            #print msg
                        i += 1
                        if i > len(expected):
                            break
                    out.close()
                    if len(fails) > 0:
                        err = (c_name, len(fails), fails[0])
                        self.fail("client %s outfile had %d mismatches. first=%s" % err)
                    elif len(expected) != i:
                        err = (i, len(expected))
                        self.fail("client produced %d lines - expected %d" % err)
        finally:
            r = urllib2.urlopen("http://localhost:9233/exit")
            r.read()
            r.close()
            s_proc.join()
            if len(errs) > 0:
                self.fail("\n".join(errs))

if __name__ == "__main__":
    unittest.main()
<commit_msg>Add Java server back to conformance test.  Passes batch test.<commit_after>#!/usr/bin/env python

from subprocess import Popen, PIPE
import time
import threading
import unittest
import os
import urllib2
import codecs
try:
    import json
except:
    import simplejson as json

# Barrister conformance test runner

home = os.environ["HOME"]

#
# Java config
#
m2_jackson = "%s/.m2/repository/org/codehaus/jackson" % home
jackson_deps = [ "jackson-mapper-asl/1.9.4/jackson-mapper-asl-1.9.4.jar",
                 "jackson-core-asl/1.9.4/jackson-core-asl-1.9.4.jar" ]
barrister_java = "../../barrister-java"
if os.environ.has_key("BARRISTER_JAVA"):
    barrister_java = os.environ["BARRISTER_JAVA"]
java_cp = "%s/target/classes:%s/conform/target/classes" % (barrister_java, barrister_java)
for d in jackson_deps:
    java_cp += ":%s/%s" % (m2_jackson, d)
winstone_jar = "%s/conform/lib/winstone-0.9.10.jar" % barrister_java
java_war = "%s/conform/target/barrister-conform-test.war" % barrister_java

#
# Clients to run against each server
#
clients = [
    # format: name, command line
    [ "python-client", ["python", "client.py"] ],
    [ "java-client", ["java", "-cp", java_cp, "com.bitmechanic.barrister.conform.Client" ] ]
]

verbose = os.environ.has_key('CONFORM_VERBOSE')
def log(msg):
    if verbose:
        print msg.strip()

class Runner(threading.Thread):

    def __init__(self, name, cmd):
        super(Runner, self).__init__()
        self.name = name
        self.cmd = cmd
        self.exit_code = None
        self.proc = None

    def run(self):
        log("[%s] Starting process: %s" % (self.name, " ".join(self.cmd)))
        self.proc = Popen(self.cmd, stdout=PIPE, stderr=PIPE, close_fds=False,
                          shell=False)
        out, err = self.proc.communicate()
        for line in out.split("\n"):
            log("[%s out] %s" % (self.name, line))
        for line in err.split("\n"):
            log("[%s err] %s" % (self.name, line))
        self.exit_code = self.proc.poll()

class ConformTest(unittest.TestCase):

    def test_python_server(self):
        cmd = ["python", "flask_server.py", "conform.json"]
        self._test_server(1, "python-flask", cmd)

    def test_java_server(self):
        cmd = ["java", "-DidlJson=conform.json", "-jar", winstone_jar,
               "--httpPort=9233", java_war ]
        self._test_server(3, "java", cmd)

    def _test_server(self, sleep_time, s_name, s_cmd):
        errs = [ ]
        expected = [ ]
        infile = "conform.in"
        inf = codecs.open(infile, "r", "utf-8")
        for line in inf:
            line = line.strip()
            if line != "" and line.find("#") != 0 and len(line.split("|")) == 5:
                expected.append(line)
        inf.close()
        try:
            s_proc = Runner(s_name, s_cmd)
            s_proc.start()
            time.sleep(sleep_time)
            for c_name, c_cmd in clients:
                print "Testing client '%s' vs server '%s'" % (c_name, s_name)
                outfile = "%s-to-%s.out" % (c_name, s_name)
                c_cmd.extend([infile, outfile])
                c_proc = Runner(c_name, c_cmd)
                c_proc.start()
                c_proc.join()
                if c_proc.exit_code != 0:
                    errs.append("client %s failed - %d" % (c_name, c_proc.exit_code))
                else:
                    out = codecs.open(outfile, "r", "utf-8")
                    i = 0
                    fails = []
                    for line in out:
                        e_cols = expected[i].split("|")
                        l_cols = line.split("|")
                        match = True
                        #print expected[i]
                        #print line
                        if len(e_cols) != len(l_cols):
                            match = False
                        elif e_cols[3] != l_cols[3]:
                            match = False
                        elif json.loads(e_cols[4]) != json.loads(l_cols[4]):
                            match = False
                        if not match:
                            msg = "line %d: %s != %s" % (i, expected[i], line)
                            fails.append(msg)
                            #print msg
                        i += 1
                        if i > len(expected):
                            break
                    out.close()
                    if len(fails) > 0:
                        err = (outfile, len(fails), fails[0])
                        self.fail("%s had %d mismatches. first=%s" % err)
                    elif len(expected) != i:
                        err = (i, len(expected))
                        self.fail("client produced %d lines - expected %d" % err)
        finally:
            r = urllib2.urlopen("http://localhost:9233/exit")
            r.read()
            r.close()
            s_proc.join()
            if len(errs) > 0:
                self.fail("\n".join(errs))

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function
from __future__ import unicode_literals, division

import argparse
import logging
import sqlite3
import subprocess
import sys
from datetime import datetime
from time import sleep

from six.moves import input

logger = logging.getLogger(__name__)

def parse_date(s, default=None):
    if not s:
        return default

    try:
        return datetime.strptime(s, "%Y-%m-%d").date().replace(day=1)
    except ValueError:
        msg = "Not a valid date: '{0}'.".format(s)
        raise argparse.ArgumentTypeError(msg)

class Database(object):
    def __init__(self, db_path):
        self.db_path = db_path

    def execute(self, query, *params):
        with self.db:
            res = self.db.execute(query, params)
            return res.fetchall()

    def get_table(self, table_name):
        rows = self.execute('select * from tables where source_table = ?', table_name)
        if not rows:
            raise Exception('No table found with name "{}"'.format(table_name))

        row = rows[0]
        if row['migrated']:
            raise Exception('Table "{}" has already been migrated'.format(table_name))

        return row['source_table'], row['target_table']

    def get_tables(self, start_date=None, end_date=None):
        params = []
        filters = []
        if start_date and end_date:
            filters.append('date is not null')

        if start_date:
            if not end_date:
                filters.append("date is null OR date >= ?")
            else:
                filters.append("date >= ?")
            params.append(start_date)

        if end_date:
            filters.append("date < ?")
            params.append(end_date)

        query = 'SELECT * FROM tables WHERE migrated is null'
        if filters:
            query += ' and ({})'.format(' and '.join(filters))
        return [
            (row['source_table'], row['target_table'])
            for row in self.execute(query, *params)
        ]

    def mark_migrated(self, tables):
        with self.db:
            for table in tables:
                self.db.execute('update tables set migrated = 1 where source_table = ?', [table])

    def __enter__(self):
        self.db = sqlite3.connect(self.db_path)
        self.db.row_factory = sqlite3.Row

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.db.close()

class TableMigration(object):
    def __init__(self, table, command):
        self.table = table
        self.command = command
        self.process = None

    def run(self):
        print(self.command)
        self.process = subprocess.Popen(self.command, shell=True)
        return self

    def poll(self):
        return self.process.poll()

    def wait(self):
        return self.process.wait()

    def __repr__(self):
        return 'TableMigration({})'.format(self.table)

class MigrationPool(object):
    def __init__(self, max_size):
        self.max_size = max_size
        self._pool = set()
        self.errors = []

    def run(self, process):
        if self.errors:
            return

        complete = self.wait_for_capacity()
        self._pool.add(process.run())
        return complete

    def wait_for_capacity(self):
        if len(self._pool) < self.max_size:
            return

        complete = []
        while not complete:
            sleep(0.5)
            for process in frozenset(self._pool):
                code = process.poll()
                if code is not None:
                    self._pool.remove(process)
                    complete.append(process)
                    if code != 0:
                        self.errors.append(process)
        return complete

    def wait(self):
        complete = []
        for process in frozenset(self._pool):
            code = process.poll()
            if code is not None:
                if code > 0:
                    self.errors.append(process)
                else:
                    complete.append(process)
            else:
                process.wait()
        return complete

class Migrator(object):
    def __init__(self, db_path, source_db, source_host, source_user,
                 target_db, target_host, target_user,
                 start_date, end_date,
                 start_table, only_table, confirm, dry_run):
        self.db_path = db_path
        self.source_db = source_db
        self.source_host = source_host
        self.source_user = source_user
        self.target_db = target_db
        self.target_host = target_host
        self.target_user = target_user
        self.start_date = start_date
        self.end_date = end_date
        self.start_table = start_table
        self.only_table = only_table
        self.confirm = confirm
        self.dry_run = dry_run

        self.db = Database(self.db_path)

    def run(self):
        with self.db:
            if self.only_table:
                table = self.db.get_table(self.only_table)
                tables = [table]
            else:
                tables = self.db.get_tables(self.start_date, self.end_date)

            if not tables:
                raise Exception("No table to migrate")

            if self.dry_run or _confirm('Preparing to migrate {} tables.'.format(len(tables))):
                self.migrate_tables(tables)

    def migrate_tables(self, tables):
        table_sizes = None
        total_size = None
        progress = float(0)

        if not self.dry_run:
            table_sizes = get_table_sizes(self.source_db, self.source_host, self.source_user)

            if table_sizes is not None:
                source_tables = {source_table for source_table, target_table in tables}
                total_size = sum([size for table, size in table_sizes.items() if table in source_tables])

        def _update_progress(tables, progress):
            if table_sizes:
                for table in tables:
                    progress += table_sizes[table]
                print('\nProgress: {:.1f}% data ({} of {}), {:.1f}% tables ({} of {})\n'.format(
                    (100 * progress / total_size) if total_size else 100, int(progress), total_size,
                    100 * float(table_index) / total_tables, table_index, total_tables,
                ))
            return progress

        commands = self.get_dump_load_commands(tables)

        total_tables = len(tables)
        pool = MigrationPool(5)
        for table_index, [source_table, target_table, cmd] in enumerate(commands):
            if not self.dry_run and (not self.confirm or _confirm('Migrate {} to {}'.format(source_table, target_table))):
                complete = pool.run(TableMigration(source_table, cmd))
                if complete:
                    complete_tables = [m.table for m in complete if m not in pool.errors]
                    self.db.mark_migrated(complete_tables)
                    progress = _update_progress(complete_tables, progress)
            else:
                print(cmd)

        complete = pool.wait()
        self.db.mark_migrated([m.table for m in complete])

        if pool.errors:
            print("\nErrors encountered:")
            for process in pool.errors:
                print("\t{}".format(process.table))
            sys.exit(1)

    def get_dump_load_commands(self, tables):
        for source_table, target_table in tables:
            cmd = 'pg_dump -h {} -U {} -t {} {} --data-only --no-acl'.format(
                self.source_host, self.source_user, source_table, self.source_db
            )
            if target_table:
                cmd += ' | sed "s/\\"\\?{}\\"\\?/\\"{}\\"/g"'.format(source_table, target_table)
            cmd += ' | psql -h {} -U {} -v ON_ERROR_STOP=1 {} --single-transaction'.format(
                self.target_host, self.target_user, self.target_db
            )

            yield source_table, target_table, cmd

def get_table_sizes(source_db, source_host, source_user):
    try:
        from sqlalchemy import create_engine
    except ImportError:
        print('sqlalchemy not installed. Progress not supported.')
        return

    engine = create_engine("postgresql://{}:@{}/{}".format(source_user, source_host, source_db))
    with engine.begin() as conn:
        res = conn.execute("select relname, n_live_tup from pg_stat_user_tables")
        return {
            row.relname: row.n_live_tup
            for row in res
        }

def filter_tables_by_date(tables, start_date, end_date):
    return [
        (source_table, target_table) for source_table, table_date, target_table in tables
        if (not start_date or table_date >= start_date) and (not end_date or table_date < end_date)
    ]

def _confirm(msg):
    confirm_update = input(msg + ' [yes / no] ')
    return confirm_update == 'yes'

def main():
    parser = argparse.ArgumentParser(description="""
        Migrate DB tables from one DB to another using pg_dump.
        Compainion to custom/icds_reports/management_commands/generate_migration_tables.py
    """)
    parser.add_argument(
        'db_path',
        help='Path to sqlite DB containing list of tables to migrate'
    )
    parser.add_argument(
        '-D', '--source-db',
        required=True,
        help='Name for source database'
    )
    parser.add_argument(
        '-O', '--source-host',
        required=True,
        help='Name for source database'
    )
    parser.add_argument(
        '-U', '--source-user',
        required=True,
        help='Name for source database'
    )
    parser.add_argument(
        '-d', '--target-db',
        required=True,
        help='Name for target database'
    )
    parser.add_argument(
        '-o', '--target-host',
        required=True,
        help='Name for target database'
    )
    parser.add_argument(
        '-u', '--target-user',
        required=True,
        help='PG user to connect to target DB as. This user should be able to connect to the target'
             'DB without a password.',
    )
    parser.add_argument(
        '--start-date',
        type=parse_date,
        help='Only migrate tables with date on or after this date. Format YYYY-MM-DD',
    )
    parser.add_argument(
        '--end-date',
        type=parse_date,
        help='Only migrate tables with date before this date. Format YYYY-MM-DD',
    )
    parser.add_argument(
        '--start-after-table',
        help='Skip all tables up to and including this one',
    )
    parser.add_argument(
        '--table',
        help='Only migrate this table',
    )
    parser.add_argument(
        '--confirm',
        action='store_true',
        help='Confirm before each table.',
    )
    parser.add_argument(
        '--dry-run',
        action='store_true',
        help='Only output the commands.',
    )

    args = parser.parse_args()

    migrator = Migrator(
        args.db_path, args.source_db, args.source_host, args.source_user,
        args.target_db, args.target_host, args.target_user,
        args.start_date, args.end_date,
        args.start_after_table, args.table, args.confirm, args.dry_run
    )
    migrator.run()

if __name__ == "__main__":
    main()
<commit_msg>configurable parallel<commit_after>from __future__ import absolute_import, print_function
from __future__ import unicode_literals, division

import argparse
import logging
import sqlite3
import subprocess
import sys
from datetime import datetime
from time import sleep

from six.moves import input

logger = logging.getLogger(__name__)

def parse_date(s, default=None):
    if not s:
        return default

    try:
        return datetime.strptime(s, "%Y-%m-%d").date().replace(day=1)
    except ValueError:
        msg = "Not a valid date: '{0}'.".format(s)
        raise argparse.ArgumentTypeError(msg)

class Database(object):
    def __init__(self, db_path):
        self.db_path = db_path

    def execute(self, query, *params):
        with self.db:
            res = self.db.execute(query, params)
            return res.fetchall()

    def get_table(self, table_name):
        rows = self.execute('select * from tables where source_table = ?', table_name)
        if not rows:
            raise Exception('No table found with name "{}"'.format(table_name))

        row = rows[0]
        if row['migrated']:
            raise Exception('Table "{}" has already been migrated'.format(table_name))

        return row['source_table'], row['target_table']

    def get_tables(self, start_date=None, end_date=None):
        params = []
        filters = []
        if start_date and end_date:
            filters.append('date is not null')

        if start_date:
            if not end_date:
                filters.append("date is null OR date >= ?")
            else:
                filters.append("date >= ?")
            params.append(start_date)

        if end_date:
            filters.append("date < ?")
            params.append(end_date)

        query = 'SELECT * FROM tables WHERE migrated is null'
        if filters:
            query += ' and ({})'.format(' and '.join(filters))
        return [
            (row['source_table'], row['target_table'])
            for row in self.execute(query, *params)
        ]

    def mark_migrated(self, tables):
        with self.db:
            for table in tables:
                self.db.execute('update tables set migrated = 1 where source_table = ?', [table])

    def __enter__(self):
        self.db = sqlite3.connect(self.db_path)
        self.db.row_factory = sqlite3.Row

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.db.close()

class TableMigration(object):
    def __init__(self, table, command):
        self.table = table
        self.command = command
        self.process = None

    def run(self):
        print(self.command)
        self.process = subprocess.Popen(self.command, shell=True)
        return self

    def poll(self):
        return self.process.poll()

    def wait(self):
        return self.process.wait()

    def __repr__(self):
        return 'TableMigration({})'.format(self.table)

class MigrationPool(object):
    def __init__(self, max_size):
        self.max_size = max_size
        self._pool = set()
        self.errors = []

    def run(self, process):
        if self.errors:
            return

        complete = self.wait_for_capacity()
        self._pool.add(process.run())
        return complete

    def wait_for_capacity(self):
        if len(self._pool) < self.max_size:
            return

        complete = []
        while not complete:
            sleep(0.5)
            for process in frozenset(self._pool):
                code = process.poll()
                if code is not None:
                    self._pool.remove(process)
                    complete.append(process)
                    if code != 0:
                        self.errors.append(process)
        return complete

    def wait(self):
        complete = []
        for process in frozenset(self._pool):
            code = process.poll()
            if code is not None:
                if code > 0:
                    self.errors.append(process)
                else:
                    complete.append(process)
            else:
                process.wait()
        return complete

class Migrator(object):
    def __init__(self, db_path, source_db, source_host, source_user,
                 target_db, target_host, target_user,
                 start_date, end_date,
                 start_table, only_table, confirm, dry_run):
        self.db_path = db_path
        self.source_db = source_db
        self.source_host = source_host
        self.source_user = source_user
        self.target_db = target_db
        self.target_host = target_host
        self.target_user = target_user
        self.start_date = start_date
        self.end_date = end_date
        self.start_table = start_table
        self.only_table = only_table
        self.confirm = confirm
        self.dry_run = dry_run

        self.db = Database(self.db_path)

    def run(self, max_concurrent):
        with self.db:
            if self.only_table:
                table = self.db.get_table(self.only_table)
                tables = [table]
            else:
                tables = self.db.get_tables(self.start_date, self.end_date)

            if not tables:
                raise Exception("No table to migrate")

            if self.dry_run or _confirm('Preparing to migrate {} tables.'.format(len(tables))):
                self.migrate_tables(tables, max_concurrent)

    def migrate_tables(self, tables, max_concurrent):
        table_sizes = None
        total_size = None
        progress = float(0)

        if not self.dry_run:
            table_sizes = get_table_sizes(self.source_db, self.source_host, self.source_user)

            if table_sizes is not None:
                source_tables = {source_table for source_table, target_table in tables}
                total_size = sum([size for table, size in table_sizes.items() if table in source_tables])

        def _update_progress(tables, progress):
            if table_sizes:
                for table in tables:
                    progress += table_sizes[table]
                print('\nProgress: {:.1f}% data ({} of {}), {:.1f}% tables ({} of {})\n'.format(
                    (100 * progress / total_size) if total_size else 100, int(progress), total_size,
                    100 * float(table_index) / total_tables, table_index, total_tables,
                ))
            return progress

        commands = self.get_dump_load_commands(tables)

        total_tables = len(tables)
        pool = MigrationPool(max_concurrent)
        for table_index, [source_table, target_table, cmd] in enumerate(commands):
            if not self.dry_run and (not self.confirm or _confirm('Migrate {} to {}'.format(source_table, target_table))):
                complete = pool.run(TableMigration(source_table, cmd))
                if complete:
                    complete_tables = [m.table for m in complete if m not in pool.errors]
                    self.db.mark_migrated(complete_tables)
                    progress = _update_progress(complete_tables, progress)
            else:
                print(cmd)

        complete = pool.wait()
        self.db.mark_migrated([m.table for m in complete])

        if pool.errors:
            print("\nErrors encountered:")
            for process in pool.errors:
                print("\t{}".format(process.table))
            sys.exit(1)

    def get_dump_load_commands(self, tables):
        for source_table, target_table in tables:
            cmd = 'pg_dump -h {} -U {} -t {} {} --data-only --no-acl'.format(
                self.source_host, self.source_user, source_table, self.source_db
            )
            if target_table:
                cmd += ' | sed "s/\\"\\?{}\\"\\?/\\"{}\\"/g"'.format(source_table, target_table)
            cmd += ' | psql -h {} -U {} -v ON_ERROR_STOP=1 {} --single-transaction'.format(
                self.target_host, self.target_user, self.target_db
            )

            yield source_table, target_table, cmd

def get_table_sizes(source_db, source_host, source_user):
    try:
        from sqlalchemy import create_engine
    except ImportError:
        print('sqlalchemy not installed. Progress not supported.')
        return

    engine = create_engine("postgresql://{}:@{}/{}".format(source_user, source_host, source_db))
    with engine.begin() as conn:
        res = conn.execute("select relname, n_live_tup from pg_stat_user_tables")
        return {
            row.relname: row.n_live_tup
            for row in res
        }

def filter_tables_by_date(tables, start_date, end_date):
    return [
        (source_table, target_table) for source_table, table_date, target_table in tables
        if (not start_date or table_date >= start_date) and (not end_date or table_date < end_date)
    ]

def _confirm(msg):
    confirm_update = input(msg + ' [yes / no] ')
    return confirm_update == 'yes'

def main():
    parser = argparse.ArgumentParser(description="""
        Migrate DB tables from one DB to another using pg_dump.
        Compainion to custom/icds_reports/management_commands/generate_migration_tables.py
    """)
    parser.add_argument(
        'db_path',
        help='Path to sqlite DB containing list of tables to migrate'
    )
    parser.add_argument(
        '-D', '--source-db',
        required=True,
        help='Name for source database'
    )
    parser.add_argument(
        '-O', '--source-host',
        required=True,
        help='Name for source database'
    )
    parser.add_argument(
        '-U', '--source-user',
        required=True,
        help='Name for source database'
    )
    parser.add_argument(
        '-d', '--target-db',
        required=True,
        help='Name for target database'
    )
    parser.add_argument(
        '-o', '--target-host',
        required=True,
        help='Name for target database'
    )
    parser.add_argument(
        '-u', '--target-user',
        required=True,
        help='PG user to connect to target DB as. This user should be able to connect to the target'
             'DB without a password.',
    )
    parser.add_argument(
        '--start-date',
        type=parse_date,
        help='Only migrate tables with date on or after this date. Format YYYY-MM-DD',
    )
    parser.add_argument(
        '--end-date',
        type=parse_date,
        help='Only migrate tables with date before this date. Format YYYY-MM-DD',
    )
    parser.add_argument(
        '--start-after-table',
        help='Skip all tables up to and including this one',
    )
    parser.add_argument(
        '--table',
        help='Only migrate this table',
    )
    parser.add_argument(
        '--confirm',
        action='store_true',
        help='Confirm before each table.',
    )
    parser.add_argument(
        '--dry-run',
        action='store_true',
        help='Only output the commands.',
    )
    parser.add_argument(
        '--parallel',
        type=int,
        default=1,
        help='How many commands to run in parallel',
    )

    args = parser.parse_args()

    migrator = Migrator(
        args.db_path, args.source_db, args.source_host, args.source_user,
        args.target_db, args.target_host, args.target_user,
        args.start_date, args.end_date,
        args.start_after_table, args.table, args.confirm, args.dry_run
    )
    migrator.run(args.parallel)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># $Id$

import sys
import os
import urllib, urlparse
from gtk import *
from gnome.ui import *
from gtkhtml import *
import cStringIO, threading, Queue, time
import pycurl

# URL history
history = []
# Links for 'forward'
forward = []
# Number of concurrent connections to the web-server
NUM_THREADS = 4

# About
def about(button):
    GnomeAbout('GtkHTML Test with PycURL', '',
                'License GPL2',
                ['Original code by Matt Wilson, modified by Kjetil Jacobsen'],
                ('This is a useless application demonstrating the '
                'GtkHTML widget with Python and PycURL.')).show()

# HTML template for reporting internal errors
internal_error = """

<title>Error</title>

Error

%s

"""

# Worker threads downloads objects and passes them to the renderer
class WorkerThread(threading.Thread):

    def __init__(self, queue, render):
        threading.Thread.__init__(self)
        self.queue = queue      # Download request queue
        self.render = render    # Render output queue

    def run(self):
        curl = pycurl.Curl()
        curl.setopt(pycurl.FOLLOWLOCATION, 1)
        curl.setopt(pycurl.MAXREDIRS, 5)
        curl.setopt(pycurl.HTTPHEADER, ["User-Agent: GtkHTML/PycURL demo browser"])
        while 1:
            url, handle = self.queue.get()
            if url == None:
                curl.close()
                raise SystemExit
            b = cStringIO.StringIO()
            curl.setopt(pycurl.WRITEFUNCTION, b.write)
            curl.setopt(pycurl.URL, url)
            try:
                curl.perform()
            except pycurl.error, msg:
                b.write(internal_error % msg[1])
            except:
                msg = "Error retrieving URL: %s" % url
                b.write(internal_error % msg)
            # Flag empty documents to the renderer
            if b.tell() == 0:
                b.close()
                b = None
            # Enqueue the document on the rendering pipeline
            self.render.append((b, handle))

# Main rendering window, handles gtk events and sends requests to worker threads
class HtmlWindow(GtkHTML):

    def __init__(self):
        GtkHTML.__init__(self)
        self.queue = Queue.Queue()
        self.render = []
        self.threads = []
        for num_threads in range(NUM_THREADS):
            t = WorkerThread(self.queue, self.render)
            t.start()
            self.threads.append(t)

    def mainquit(self, *args):
        # Send a 'terminate' message to the worker threads
        for t in self.threads:
            t.queue.put((None, None))
        mainquit()

    def load_url(self, html, url):
        t1 = time.time()
        self.num_obj = 0
        if history: url = urllib.basejoin(history[-1], url)
        history.append(url)
        html.load_empty()
        handle = html.begin()
        url = url.strip()
        self.request_url(html, url, handle)
        self.urlentry.set_text(url)
        # Render incoming objects
        while self.num_obj > 0:
            if len(self.render) == 0:
                mainiteration(0)
                continue
            self.num_obj -= 1
            buf, handle = self.render.pop(0)
            if buf != None:
                html.write(handle, buf.getvalue())
                buf.close()
            html.end(handle, HTML_STREAM_OK)
        t2 = time.time()
        self.statusbar.set_text("Done (%.3f seconds)" % (t2-t1))

    def submit(self, html, method, path, params):
        print 'Submit is not supported yet'
        print method, path, params

    def request_url(self, html, url, handle):
        url = urllib.basejoin(history[-1], url)
        self.statusbar.set_text("Requesting URL: %s" % url)
        self.queue.put((url, handle))
        self.num_obj += 1

    def entry_activate(self, entry, html):
        url = entry.get_text()
        self.load_url(html, url)
        del forward[:]

    def do_back(self, _b):
        forward.append(history[-1])
        del history[-1]
        url = history[-1]
        del history[-1]
        self.load_url(html, url)

    def do_forward(self, _b):
        if len(forward) == 0: return
        self.load_url(html, forward[-1])
        del forward[-1]

    def do_reload(self, _b):
        if len(history) == 0: return
        url = history[-1]
        del history[-1]
        self.load_url(html, url)

# Setup windows and menus
html = HtmlWindow()

file_menu = [
    UIINFO_ITEM_STOCK('Quit', None, html.mainquit, STOCK_MENU_QUIT),
]
help_menu = [
    UIINFO_ITEM_STOCK('About...', None, about, STOCK_MENU_ABOUT),
]
menus = [
    UIINFO_SUBTREE('File', file_menu),
    UIINFO_SUBTREE('Help', help_menu)
]

toolbar = [
    UIINFO_ITEM_STOCK('Back', 'Previous page', html.do_back, STOCK_MENU_BACK),
    UIINFO_ITEM_STOCK('Forward', 'Next page', html.do_forward, STOCK_MENU_FORWARD),
    UIINFO_ITEM_STOCK('Reload', 'Reload current page', html.do_reload, STOCK_MENU_REFRESH)
]

win = GnomeApp("html_demo", "Python GtkHTML Test")
win.set_wmclass("gtk_html_test", "GtkHTMLTest")
win.connect('delete_event', html.mainquit)

vbox = GtkVBox(spacing=3)
vbox.set_border_width(2)
vbox.show()
win.set_contents(vbox)

entry = GtkEntry()
html.connect('url_requested', html.request_url)
html.connect('link_clicked', html.load_url)
html.connect('submit', html.submit)
entry.connect('activate', html.entry_activate, html)
vbox.pack_start(entry, expand=FALSE)
entry.show()
html.urlentry = entry
html.set_usize(800, 600)

sw = GtkScrolledWindow()
sw.set_policy(POLICY_AUTOMATIC, POLICY_AUTOMATIC)
sw.add(html)
vbox.pack_start(sw)

sep = GtkHSeparator()
vbox.pack_start(sep, expand=FALSE)

status = GtkLabel('')
status.set_justify(JUSTIFY_LEFT)
status.set_alignment(0.0, 0.5)
win.set_statusbar(status)
win.create_menus(menus)
win.create_toolbar(toolbar)

html.statusbar = status
html.load_empty()

win.show_all()

threads_enter()
mainloop()
threads_leave()
<commit_msg>trap errors when pressing the back button<commit_after># $Id$

import sys
import os
import urllib, urlparse
from gtk import *
from gnome.ui import *
from gtkhtml import *
import cStringIO, threading, Queue, time
import pycurl

# URL history
history = []
# Links for 'forward'
forward = []
# Number of concurrent connections to the web-server
NUM_THREADS = 4

# About
def about(button):
    GnomeAbout('GtkHTML Test with PycURL', '',
                'License GPL2',
                ['Original code by Matt Wilson, modified by Kjetil Jacobsen'],
                ('This is a useless application demonstrating the '
                'GtkHTML widget with Python and PycURL.')).show()

# HTML template for reporting internal errors
internal_error = """

<title>Error</title>

Error

%s

"""

# Worker threads downloads objects and passes them to the renderer
class WorkerThread(threading.Thread):

    def __init__(self, queue, render):
        threading.Thread.__init__(self)
        self.queue = queue      # Download request queue
        self.render = render    # Render output queue

    def run(self):
        curl = pycurl.Curl()
        curl.setopt(pycurl.FOLLOWLOCATION, 1)
        curl.setopt(pycurl.MAXREDIRS, 5)
        curl.setopt(pycurl.HTTPHEADER, ["User-Agent: GtkHTML/PycURL demo browser"])
        while 1:
            url, handle = self.queue.get()
            if url == None:
                curl.close()
                raise SystemExit
            b = cStringIO.StringIO()
            curl.setopt(pycurl.WRITEFUNCTION, b.write)
            curl.setopt(pycurl.URL, url)
            try:
                curl.perform()
            except pycurl.error, msg:
                b.write(internal_error % msg[1])
            except:
                msg = "Error retrieving URL: %s" % url
                b.write(internal_error % msg)
            # Flag empty documents to the renderer
            if b.tell() == 0:
                b.close()
                b = None
            # Enqueue the document on the rendering pipeline
            self.render.append((b, handle))

# Main rendering window, handles gtk events and sends requests to worker threads
class HtmlWindow(GtkHTML):

    def __init__(self):
        GtkHTML.__init__(self)
        self.queue = Queue.Queue()
        self.render = []
        self.threads = []
        for num_threads in range(NUM_THREADS):
            t = WorkerThread(self.queue, self.render)
            t.start()
            self.threads.append(t)

    def mainquit(self, *args):
        # Send a 'terminate' message to the worker threads
        for t in self.threads:
            t.queue.put((None, None))
        mainquit()

    def load_url(self, html, url):
        t1 = time.time()
        self.num_obj = 0
        if history: url = urllib.basejoin(history[-1], url)
        history.append(url)
        html.load_empty()
        handle = html.begin()
        url = url.strip()
        self.request_url(html, url, handle)
        self.urlentry.set_text(url)
        # Render incoming objects
        while self.num_obj > 0:
            if len(self.render) == 0:
                mainiteration(0)
                continue
            self.num_obj -= 1
            buf, handle = self.render.pop(0)
            if buf != None:
                html.write(handle, buf.getvalue())
                buf.close()
            html.end(handle, HTML_STREAM_OK)
        t2 = time.time()
        self.statusbar.set_text("Done (%.3f seconds)" % (t2-t1))

    def submit(self, html, method, path, params):
        print 'Submit is not supported yet'
        print method, path, params

    def request_url(self, html, url, handle):
        url = urllib.basejoin(history[-1], url)
        self.statusbar.set_text("Requesting URL: %s" % url)
        self.queue.put((url, handle))
        self.num_obj += 1

    def entry_activate(self, entry, html):
        url = entry.get_text()
        self.load_url(html, url)
        del forward[:]

    def do_back(self, _b):
        if len(history) > 1:
            forward.append(history[-1])
            del history[-1]
            url = history[-1]
            del history[-1]
            self.load_url(html, url)

    def do_forward(self, _b):
        if len(forward) == 0: return
        self.load_url(html, forward[-1])
        del forward[-1]

    def do_reload(self, _b):
        if len(history) == 0: return
        url = history[-1]
        del history[-1]
        self.load_url(html, url)

# Setup windows and menus
html = HtmlWindow()

file_menu = [
    UIINFO_ITEM_STOCK('Quit', None, html.mainquit, STOCK_MENU_QUIT),
]
help_menu = [
    UIINFO_ITEM_STOCK('About...', None, about, STOCK_MENU_ABOUT),
]
menus = [
    UIINFO_SUBTREE('File', file_menu),
    UIINFO_SUBTREE('Help', help_menu)
]

toolbar = [
    UIINFO_ITEM_STOCK('Back', 'Previous page', html.do_back, STOCK_MENU_BACK),
    UIINFO_ITEM_STOCK('Forward', 'Next page', html.do_forward, STOCK_MENU_FORWARD),
    UIINFO_ITEM_STOCK('Reload', 'Reload current page', html.do_reload, STOCK_MENU_REFRESH)
]

win = GnomeApp("html_demo", "Python GtkHTML Test")
win.set_wmclass("gtk_html_test", "GtkHTMLTest")
win.connect('delete_event', html.mainquit)

vbox = GtkVBox(spacing=3)
vbox.set_border_width(2)
vbox.show()
win.set_contents(vbox)

entry = GtkEntry()
html.connect('url_requested', html.request_url)
html.connect('link_clicked', html.load_url)
html.connect('submit', html.submit)
entry.connect('activate', html.entry_activate, html)
vbox.pack_start(entry, expand=FALSE)
entry.show()
html.urlentry = entry
html.set_usize(800, 600)

sw = GtkScrolledWindow()
sw.set_policy(POLICY_AUTOMATIC, POLICY_AUTOMATIC)
sw.add(html)
vbox.pack_start(sw)

sep = GtkHSeparator()
vbox.pack_start(sep, expand=FALSE)

status = GtkLabel('')
status.set_justify(JUSTIFY_LEFT)
status.set_alignment(0.0, 0.5)
win.set_statusbar(status)
win.create_menus(menus)
win.create_toolbar(toolbar)

html.statusbar = status
html.load_empty()

win.show_all()

threads_enter()
mainloop()
threads_leave()
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-

# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U
#
# This file is part of FIWARE project.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
#
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
#
# For those usages not covered by the Apache version 2.0 License please
# contact with opensource@tid.es

"""Generate a summary report including the sanity status of the regions.

Usage:
  {prog} XUNIT_RESULTS_FILE

Environment:
  SANITY_CHECKS_SETTINGS            (Optional) Path to settings file

Files:
  etc/settings.json                 Default settings file
  test_results.xml                  Default xUnit results file

"""

from xml.dom import minidom
from constants import DEFAULT_SETTINGS_FILE, PROPERTIES_CONFIG_KEY_TEST_CASES, PROPERTIES_CONFIG_OPT_TEST_CASES
import os.path
import json
import sys
import re

DEFAULT_RESULTS_FILE = "test_results.xml"

ATTR_TESTS_TOTAL = "tests"
ATTR_TESTS_SKIP = "skip"
ATTR_TESTS_ERROR = "errors"
ATTR_TESTS_FAILURE = "failures"

CHILD_NODE_SKIP = "skipped"
CHILD_NODE_ERROR = "error"
CHILD_NODE_FAILURE = "failure"
CHILD_NODE_OTHER = None

TEST_STATUS_NOT_OK = "NOK"
TEST_STATUS_SKIP = "N/A"
TEST_STATUS_OK = "OK"

GLOBAL_STATUS_PARTIAL_OK = "POK"
GLOBAL_STATUS_NOT_OK = TEST_STATUS_NOT_OK
GLOBAL_STATUS_OK = TEST_STATUS_OK

class ResultAnalyzer(object):
    def __init__(self, conf, file=DEFAULT_RESULTS_FILE):
        self.conf = conf
        self.file = file
        self.dict = {}

    def get_results(self):
        """
        Parse report file (xUnit test result report) to get total results per each Region.
        """
        doc = minidom.parse(self.file)
        testsuite = doc.getElementsByTagName("testsuite")[0]

        # Print a summary of the test results
        print "[Tests: {}, Errors: {}, Failures: {}, Skipped: {}]".format(
            testsuite.getAttribute(ATTR_TESTS_TOTAL),
            testsuite.getAttribute(ATTR_TESTS_ERROR),
            testsuite.getAttribute(ATTR_TESTS_FAILURE),
            testsuite.getAttribute(ATTR_TESTS_SKIP))

        # Count errors/failures/skips
        for testcase in doc.getElementsByTagName('testcase'):
            status = TEST_STATUS_OK
            child_node_list = testcase.childNodes
            if child_node_list is not None and len(child_node_list) != 0:
                if child_node_list[0].localName in [CHILD_NODE_FAILURE, CHILD_NODE_ERROR, CHILD_NODE_OTHER]:
                    status = TEST_STATUS_NOT_OK
                elif child_node_list[0].localName == CHILD_NODE_SKIP:
                    status = TEST_STATUS_SKIP

            # Filter out the "regular" test cases (...test_<region>.TestSuite)
            testclass = testcase.getAttribute('classname')
            if re.match(".*\.test_.+\.TestSuite$", testclass):
                testpackage = testclass.split(".")[-2]
                testregion = testpackage.replace("test_", "")
                info_test = {"test_name": testcase.getAttribute('name'), "status": status}
                if testregion in self.dict:
                    self.dict[testregion].append(info_test)
                else:
                    self.dict.update({testregion: [info_test]})

    def print_results(self):
        """
        Print report to standard output
        """
        print "\n*********************************\n"
        print "REGION TEST SUMMARY REPORT: "
        for item in self.dict:
            print "\n >> {}".format(item)
            for result_value in self.dict[item]:
                print "  {status}\t {name}".format(name=result_value['test_name'], status=result_value['status'])

    def print_global_status(self):
        """
        This method will parse test results for each Region and will take into account whether all key and/or optional
        test cases are successful, according to the patterns defined in `settings.json`.

        How it works:

            * Configuration properties:

                key_test_cases_pattern = [a, b]
                opt_test_cases_pattern = [b, f, g]

            * Logic:

                key_test_cases list: It will have *ANY* test that matches with pattern
                defined in `key_test_cases_pattern`. If all the tests in this list have got
                'OK' status, then the region will pass this validation (*OK* status). i.e:

                    If test results are:
                        OK  a
                        NOK b
                        NOK c
                        OK  d
                        NOK e
                        OK  f
                        OK  g

                    > key_test_cases list will have this values: [a, b].
                    > Due to b=NOK, global region status is not complying with key test cases.

                non_opt_test_cases list: It will have *ALL* Key tests that do *NOT* match
                with pattern defined in `opt_test_cases_pattern`: All non-optional 'key' test cases.
                If all the tests in this list (non optional 'key' tests) have got *OK* status,
                then the region will paas this validation (*POK* status). This checks are only
                performed when the former one is not successful. i.e:

                    If test results are:
                        OK  a
                        NOK b
                        NOK c
                        OK  d
                        NOK e
                        OK  f
                        OK  g

                    > non_opt_test_cases list will have this values: [a]
                    > Due to a=OK, global region status is complying with this check and it will
                    have *POK* (partial OK) status.
        :return:
        """

        key_test_cases = self.conf[PROPERTIES_CONFIG_KEY_TEST_CASES]
        opt_test_cases = self.conf[PROPERTIES_CONFIG_OPT_TEST_CASES]

        # dictionary holding global status according either key or optional test cases
        global_status = {
            GLOBAL_STATUS_OK: {
                'caption': 'Regions satisfying all key test cases: %s' % key_test_cases,
                'empty_msg': 'NONE!!!!!!!',
                'region_list': []
            },
            GLOBAL_STATUS_PARTIAL_OK: {
                'caption': 'Regions only failing in optional test cases: %s' % opt_test_cases,
                'empty_msg': 'N/A',
                'region_list': []
            }
        }

        # check status
        key_test_cases_patterns = [re.compile(item) for item in key_test_cases]
        opt_test_cases_patterns = [re.compile(item) for item in opt_test_cases]
        for region, results in self.dict.iteritems():
            key_test_cases = [
                item for item in results
                if any(pattern.match(item['test_name']) for pattern in key_test_cases_patterns)
            ]
            non_opt_test_cases = [
                item for item in key_test_cases
                if all(not pattern.match(item['test_name']) for pattern in opt_test_cases_patterns)
            ]

            if all(item['status'] == TEST_STATUS_OK for item in key_test_cases):
                global_status[GLOBAL_STATUS_OK]['region_list'].append(region)
            elif all(item['status'] == TEST_STATUS_OK for item in non_opt_test_cases):
                global_status[GLOBAL_STATUS_PARTIAL_OK]['region_list'].append(region)

        # print status
        print "\nREGION GLOBAL STATUS"
        for status in [GLOBAL_STATUS_OK, GLOBAL_STATUS_PARTIAL_OK]:
            region_list = global_status[status]['region_list']
            print "\n", global_status[status]['caption']
            print " >> %s" % ", ".join(region_list) if len(region_list) else " %s" % global_status[status]['empty_msg']

if __name__ == "__main__":
    if len(sys.argv) != 2:
        usage = re.findall(r'.*{prog}.*', __doc__)[0].format(prog=os.path.basename(__file__)).strip()
        print "Usage: %s" % usage
        sys.exit(-1)

    results_file = sys.argv[1]

    parentdir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
    settings_file = os.environ.get('SANITY_CHECKS_SETTINGS', os.path.join(parentdir, DEFAULT_SETTINGS_FILE))
    with open(settings_file) as settings:
        try:
            conf = json.load(settings)
        except Exception as e:
            print "Error parsing config file '{}': {}".format(settings_file, e)
            sys.exit(-1)

    checker = ResultAnalyzer(conf, results_file)
    checker.get_results()
    checker.print_global_status()
    checker.print_results()
<commit_msg>Fix region name parsing when analyzing xUnit results<commit_after>#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-

# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U
#
# This file is part of FIWARE project.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
#
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
#
# For those usages not covered by the Apache version 2.0 License please
# contact with opensource@tid.es

"""Generate a summary report including the sanity status of the regions.

Usage:
  {prog} XUNIT_RESULTS_FILE

Environment:
  SANITY_CHECKS_SETTINGS            (Optional) Path to settings file

Files:
  etc/settings.json                 Default settings file
  test_results.xml                  Default xUnit results file

"""

from xml.dom import minidom
from constants import DEFAULT_SETTINGS_FILE, PROPERTIES_CONFIG_KEY_TEST_CASES, PROPERTIES_CONFIG_OPT_TEST_CASES
import os.path
import json
import sys
import re

DEFAULT_RESULTS_FILE = "test_results.xml"

ATTR_TESTS_TOTAL = "tests"
ATTR_TESTS_SKIP = "skip"
ATTR_TESTS_ERROR = "errors"
ATTR_TESTS_FAILURE = "failures"

CHILD_NODE_SKIP = "skipped"
CHILD_NODE_ERROR = "error"
CHILD_NODE_FAILURE = "failure"
CHILD_NODE_OTHER = None

TEST_STATUS_NOT_OK = "NOK"
TEST_STATUS_SKIP = "N/A"
TEST_STATUS_OK = "OK"

GLOBAL_STATUS_PARTIAL_OK = "POK"
GLOBAL_STATUS_NOT_OK = TEST_STATUS_NOT_OK
GLOBAL_STATUS_OK = TEST_STATUS_OK

class ResultAnalyzer(object):

    def __init__(self, conf, file=DEFAULT_RESULTS_FILE):
        self.conf = conf
        self.file = file
        self.dict = {}

    def get_results(self):
        """
        Parse report file (xUnit test result report) to get total results per each Region.
        """
        doc = minidom.parse(self.file)
        testsuite = doc.getElementsByTagName("testsuite")[0]

        # Print a summary of the test results
        print "[Tests: {}, Errors: {}, Failures: {}, Skipped: {}]".format(
            testsuite.getAttribute(ATTR_TESTS_TOTAL),
            testsuite.getAttribute(ATTR_TESTS_ERROR),
            testsuite.getAttribute(ATTR_TESTS_FAILURE),
            testsuite.getAttribute(ATTR_TESTS_SKIP))

        # Count errors/failures/skips
        for testcase in doc.getElementsByTagName('testcase'):
            status = TEST_STATUS_OK
            child_node_list = testcase.childNodes
            if child_node_list is not None and len(child_node_list) != 0:
                if child_node_list[0].localName in [CHILD_NODE_FAILURE, CHILD_NODE_ERROR, CHILD_NODE_OTHER]:
                    status = TEST_STATUS_NOT_OK
                elif child_node_list[0].localName == CHILD_NODE_SKIP:
                    status = TEST_STATUS_SKIP

            # Filter out the "regular" test cases (__main__.<Region>)
            testclass = testcase.getAttribute('classname')
            if re.match("^__main__\.[A-Z].+$", testclass):
                testregion = testclass.split(".")[1]
                info_test = {"test_name": testcase.getAttribute('name'), "status": status}
                if testregion in self.dict:
                    self.dict[testregion].append(info_test)
                else:
                    self.dict.update({testregion: [info_test]})

    def print_results(self):
        """
        Print report to standard output
        """
        print "\n*********************************\n"
        print "REGION TEST SUMMARY REPORT: "
        for item in self.dict:
            print "\n >> {}".format(item)
            for result_value in self.dict[item]:
                print "  {status}\t {name}".format(name=result_value['test_name'], status=result_value['status'])

    def print_global_status(self):
        """
        This method will parse test results for each Region and will take into account whether all key and/or optional
        test cases are successful, according to the patterns defined in `settings.json`.

        How it works:

            * Configuration properties:

                key_test_cases_pattern = [a, b]
                opt_test_cases_pattern = [b, f, g]

            * Logic:

                key_test_cases list: It will have *ANY* test that matches with pattern
                defined in `key_test_cases_pattern`. If all the tests in this list have got
                'OK' status, then the region will pass this validation (*OK* status). i.e:

                    If test results are:
                        OK  a
                        NOK b
                        NOK c
                        OK  d
                        NOK e
                        OK  f
                        OK  g

                    > key_test_cases list will have this values: [a, b].
                    > Due to b=NOK, global region status is not complying with key test cases.

                non_opt_test_cases list: It will have *ALL* Key tests that do *NOT* match
                with pattern defined in `opt_test_cases_pattern`: All non-optional 'key' test cases.
                If all the tests in this list (non optional 'key' tests) have got *OK* status,
                then the region will paas this validation (*POK* status). This checks are only
                performed when the former one is not successful. i.e:

                    If test results are:
                        OK  a
                        NOK b
                        NOK c
                        OK  d
                        NOK e
                        OK  f
                        OK  g

                    > non_opt_test_cases list will have this values: [a]
                    > Due to a=OK, global region status is complying with this check and it will
                    have *POK* (partial OK) status.
        :return:
        """

        key_test_cases = self.conf[PROPERTIES_CONFIG_KEY_TEST_CASES]
        opt_test_cases = self.conf[PROPERTIES_CONFIG_OPT_TEST_CASES]

        # dictionary holding global status according either key or optional test cases
        global_status = {
            GLOBAL_STATUS_OK: {
                'caption': 'Regions satisfying all key test cases: %s' % key_test_cases,
                'empty_msg': 'NONE!!!!!!!',
                'region_list': []
            },
            GLOBAL_STATUS_PARTIAL_OK: {
                'caption': 'Regions only failing in optional test cases: %s' % opt_test_cases,
                'empty_msg': 'N/A',
                'region_list': []
            }
        }

        # check status
        key_test_cases_patterns = [re.compile(item) for item in key_test_cases]
        opt_test_cases_patterns = [re.compile(item) for item in opt_test_cases]
        for region, results in self.dict.iteritems():
            key_test_cases = [
                item for item in results
                if any(pattern.match(item['test_name']) for pattern in key_test_cases_patterns)
            ]
            non_opt_test_cases = [
                item for item in key_test_cases
                if all(not pattern.match(item['test_name']) for pattern in opt_test_cases_patterns)
            ]

            if all(item['status'] == TEST_STATUS_OK for item in key_test_cases):
                global_status[GLOBAL_STATUS_OK]['region_list'].append(region)
            elif all(item['status'] == TEST_STATUS_OK for item in non_opt_test_cases):
                global_status[GLOBAL_STATUS_PARTIAL_OK]['region_list'].append(region)

        # print status
        print "\nREGION GLOBAL STATUS"
        for status in [GLOBAL_STATUS_OK, GLOBAL_STATUS_PARTIAL_OK]:
            region_list = global_status[status]['region_list']
            print "\n", global_status[status]['caption']
            print " >> %s" % ", ".join(region_list) if len(region_list) else " %s" % global_status[status]['empty_msg']

if __name__ == "__main__":
    if len(sys.argv) != 2:
        usage = re.findall(r'.*{prog}.*', __doc__)[0].format(prog=os.path.basename(__file__)).strip()
        print "Usage: %s" % usage
        sys.exit(-1)

    results_file = sys.argv[1]

    parentdir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
    settings_file = os.environ.get('SANITY_CHECKS_SETTINGS', os.path.join(parentdir, DEFAULT_SETTINGS_FILE))
    with open(settings_file) as settings:
        try:
            conf = json.load(settings)
        except Exception as e:
            print "Error parsing config file '{}': {}".format(settings_file, e)
            sys.exit(-1)

    checker = ResultAnalyzer(conf, results_file)
    checker.get_results()
    checker.print_global_status()
    checker.print_results()
<|endoftext|>


<commit_before>"""
============================================================
Define target events based on time lag, plot evoked response
============================================================

This script shows how to define higher order events based on
time lag between reference and target events. For
illustration, we will put face stimuli presented into two
classes, that is 1) followed by an early button press
(within 590 milliseconds) and followed by a late button
press (later than 590 milliseconds). Finally, we will
visualize the evoked responses to both 'quickly-processed'
and 'slowly-processed' face stimuli.

"""
# Authors: Denis Engemann <d.engemann@fz-juelich.de>
#
# License: BSD (3-clause)

print __doc__

import mne
from mne import fiff
from mne.event import define_target_events
from mne.datasets import sample
data_path = sample.data_path('.')

###############################################################################
# Set parameters
raw_fname = data_path + '/MEG/sample/sample_audvis_filt-0-40_raw.fif'
event_fname = data_path + '/MEG/sample/sample_audvis_filt-0-40_raw-eve.fif'

#   Setup for reading the raw data
raw = fiff.Raw(raw_fname)
events = mne.read_events(event_fname)

#   Set up pick list: EEG + STI 014 - bad channels (modify to your needs)
include = []  # or stim channels ['STI 014']
exclude = raw.info['bads'] + ['EEG 053']  # bads

# pick EEG channels
picks = fiff.pick_types(raw.info, meg='mag', eeg=False, stim=False, eog=True,
                                            include=include, exclude=exclude)

###############################################################################
# Find stimulus event followed by quick button presses

reference_id = 5  # presentation of a smiley face
target_id = 32  # button press
sfreq = raw.info['sfreq']  # sampling rate
tmin = 0.1  # trials leading to very early responses will be rejected
tmax = 0.59  # ignore face stimuli followed by button press later than 590 ms
new_id = 42  # the new event id for a hit. If None, reference_id is used.
fill_na = 99  # the fill value for misses

events_, lag = define_target_events(events, reference_id, target_id,
                    sfreq, tmin, tmax, new_id, fill_na)

print events_  # The 99 indicates missing or too late button presses

# besides the events also the lag between target and reference is returned
# this could e.g. be used as parametric regressor in subsequent analyses.

print lag[lag != fill_na]  # lag in milliseconds

# #############################################################################
# Construct epochs

tmin_ = -0.2
tmax_ = 0.4
event_id = dict(early=new_id, late=fill_na)

epochs = mne.Epochs(raw, events_, event_id, tmin_,
                    tmax_, picks=picks, baseline=(None, 0),
                    reject=dict(mag=4e-12))

# average epochs and get an Evoked dataset.

early, late = [epochs[k].average() for k in event_id]

###############################################################################
# View evoked response

import pylab as pl

times = 1e3 * epochs.times  # time in milliseconds
title = 'Evoked response followed by %s button press'

pl.clf()
ax = pl.subplot(2, 1, 1)
early.plot(axes=ax)
pl.title(title % 'late')
pl.ylabel('Evoked field (fT)')
ax = pl.subplot(2, 1, 2)
late.plot(axes=ax)
pl.title(title % 'early')
pl.ylabel('Evoked field (fT)')
pl.show()
<commit_msg>COSMITS ...<commit_after>"""
============================================================
Define target events based on time lag, plot evoked response
============================================================

This script shows how to define higher order events based on
time lag between reference and target events. For
illustration, we will put face stimuli presented into two
classes, that is 1) followed by an early button press
(within 590 milliseconds) and followed by a late button
press (later than 590 milliseconds). Finally, we will
visualize the evoked responses to both 'quickly-processed'
and 'slowly-processed' face stimuli.

"""
# Authors: Denis Engemann <d.engemann@fz-juelich.de>
#
# License: BSD (3-clause)

print __doc__

import mne
from mne import fiff
from mne.event import define_target_events
from mne.datasets import sample
data_path = sample.data_path('.')

###############################################################################
# Set parameters
raw_fname = data_path + '/MEG/sample/sample_audvis_filt-0-40_raw.fif'
event_fname = data_path + '/MEG/sample/sample_audvis_filt-0-40_raw-eve.fif'

#   Setup for reading the raw data
raw = fiff.Raw(raw_fname)
events = mne.read_events(event_fname)

#   Set up pick list: EEG + STI 014 - bad channels (modify to your needs)
include = []  # or stim channels ['STI 014']
exclude = raw.info['bads'] + ['EEG 053']  # bads

# pick MEG channels
picks = fiff.pick_types(raw.info, meg='mag', eeg=False, stim=False, eog=True,
                        include=include, exclude=exclude)

###############################################################################
# Find stimulus event followed by quick button presses

reference_id = 5  # presentation of a smiley face
target_id = 32  # button press
sfreq = raw.info['sfreq']  # sampling rate
tmin = 0.1  # trials leading to very early responses will be rejected
tmax = 0.59  # ignore face stimuli followed by button press later than 590 ms
new_id = 42  # the new event id for a hit. If None, reference_id is used.
fill_na = 99  # the fill value for misses

events_, lag = define_target_events(events, reference_id, target_id,
                    sfreq, tmin, tmax, new_id, fill_na)

print events_  # The 99 indicates missing or too late button presses

# besides the events also the lag between target and reference is returned
# this could e.g. be used as parametric regressor in subsequent analyses.

print lag[lag != fill_na]  # lag in milliseconds

# #############################################################################
# Construct epochs

tmin_ = -0.2
tmax_ = 0.4
event_id = dict(early=new_id, late=fill_na)

epochs = mne.Epochs(raw, events_, event_id, tmin_,
                    tmax_, picks=picks, baseline=(None, 0),
                    reject=dict(mag=4e-12))

# average epochs and get an Evoked dataset.

early, late = [epochs[k].average() for k in event_id]

###############################################################################
# View evoked response

import pylab as pl

times = 1e3 * epochs.times  # time in milliseconds
title = 'Evoked response followed by %s button press'

pl.clf()
ax = pl.subplot(2, 1, 1)
early.plot(axes=ax)
pl.title(title % 'late')
pl.ylabel('Evoked field (fT)')
ax = pl.subplot(2, 1, 2)
late.plot(axes=ax)
pl.title(title % 'early')
pl.ylabel('Evoked field (fT)')
pl.show()
<|endoftext|>


<commit_before>"""
This module provides miscellaneous utilities.
"""

import hashlib
import functools
import os
import shutil
import sys
import imp
import logging
import tempfile
import pkg_resources

from contextlib import contextmanager
from xml.etree.ElementTree import SubElement

log = logging.getLogger(__name__)

class _memoized(object):
    """
    Decorator that caches a function's return value each time it is called.
    If called later with the same arguments, the cached value is returned, and
    not re-evaluated.
    """

    def __init__(self, func):
        self.func = func
        self.cache = {}

    def __call__(self, *args):
        try:
            return self.cache[args]
        except KeyError:
            value = self.func(*args)
            self.cache[args] = value
            return value
        except TypeError:
            # uncachable -- for instance, passing a list as an argument.
            # Better to not cache than to blow up entirely.
            return self.func(*args)

    def __repr__(self):  # pragma: no cover
        """Return the function's docstring."""
        return self.func.__doc__

    def __get__(self, obj, objtype):
        """Support instance methods."""
        return functools.partial(self.__call__, obj)

def memoized(f):
    """
    Wraps the _memoized decorator using functools so that,
    for example, the correct docstring will be used.
    """
    memoize = _memoized(f)

    @functools.wraps(f)
    def helper(*args, **kwargs):
        return memoize(*args, **kwargs)

    return helper

def extract_classpath(options):
    """
    Return the jvm classpath from a sequence of option strings.

    :param options: sequence of jvm options to search
    :returns: classpath in options
    :rtype: string
    """
    # a later classpath overwrites previous definitions so we have to search
    # from the end
    options = options[::-1]
    classpath = ''
    for i, x in enumerate(options):
        if x in ('-cp', '-classpath'):
            try:
                # cp is positioned before option because order is reversed,
                # take abs to avoid referencing from end
                cp_index = abs(i - 1)
                classpath = options[cp_index]
            except IndexError, e:
                classpath = ''
            break
    return classpath

def tree_pp(elem, level=0):
    """
    Pretty-prints an ElementTree.
    """
    i = "\n" + level * "  "
    if len(elem):
        if not elem.text or not elem.text.strip():
            elem.text = i + "  "
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
        for elem in elem:
            tree_pp(elem, level + 1)
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
    else:
        if level and (not elem.tail or not elem.tail.strip()):
            elem.tail = i

def dict2tree(elem, dictionary):
    for k, v in dictionary.items():
        if v:
            e = SubElement(elem, k)
            e.text = v

def dict2string(d, attribs=None):
    """
    PrettyPrints a dictionary as string.
    """
    s = []
    for k, v in d.items():
        if not attribs or k in attribs:
            if v:
                s.append("%s=%s" % (k, v))
    return ", ".join(s)

def sha1sum(filename, blocksize=65536):
    """
    Returns the sha1 hexdigest of a file.
    """
    hasher = hashlib.sha1()

    with open(filename, 'r') as afile:
        buf = afile.read(blocksize)
        while len(buf) > 0:
            hasher.update(buf)
            buf = afile.read(blocksize)

    return hasher.hexdigest()

@contextmanager
def tempdir(prefix='penchy-invocation', delete=False):
    """
    Execute in new created temporary directory.
    """
    fwd = os.getcwd()
    cwd = tempfile.mkdtemp(prefix=prefix)
    os.chdir(cwd)
    yield
    os.chdir(fwd)
    if delete:
        shutil.rmtree(cwd)

def find_bootstrap_client():
    """
    Returns the path of the penchy bootstrap client.
    """
    import penchy
    return pkg_resources.resource_filename('penchy',
            os.path.join('../', 'bin', 'penchy_bootstrap'))

def load_job(filename):
    """
    Loads a job.

    :param filename: filename of the job
    :type filename: string
    :param config: config file to export to job namespace
    """

    assert 'config' in sys.modules, "You have to load the penchyrc before a job"

    with disable_write_bytecode():
        job = imp.load_source('job', filename)
    log.info("Loaded job from %s" % filename)
    return job

def load_config(filename):
    """
    Loads the config module from filename. Looks
    in the current working directory as well.

    :param filename: filename of the config file
    :type filename: string
    :returns: config object
    """
    try:
        with disable_write_bytecode():
            config = imp.load_source('config', filename)
    except IOError:
        raise IOError("Config file could not be loaded from: %s" % filename)

    log.info("Loaded configuration from %s" % filename)
    return config

@contextmanager
def disable_write_bytecode():
    """
    Temporarily disables writing bytecode while executing.
    """
    old_state = sys.dont_write_bytecode
    sys.dont_write_bytecode = True
    yield
    sys.dont_write_bytecode = old_state
<commit_msg>py3: Use python3 compatible except.<commit_after>"""
This module provides miscellaneous utilities.
"""

import hashlib
import functools
import os
import shutil
import sys
import imp
import logging
import tempfile
import pkg_resources

from contextlib import contextmanager
from xml.etree.ElementTree import SubElement

log = logging.getLogger(__name__)

class _memoized(object):
    """
    Decorator that caches a function's return value each time it is called.
    If called later with the same arguments, the cached value is returned, and
    not re-evaluated.
    """

    def __init__(self, func):
        self.func = func
        self.cache = {}

    def __call__(self, *args):
        try:
            return self.cache[args]
        except KeyError:
            value = self.func(*args)
            self.cache[args] = value
            return value
        except TypeError:
            # uncachable -- for instance, passing a list as an argument.
            # Better to not cache than to blow up entirely.
            return self.func(*args)

    def __repr__(self):  # pragma: no cover
        """Return the function's docstring."""
        return self.func.__doc__

    def __get__(self, obj, objtype):
        """Support instance methods."""
        return functools.partial(self.__call__, obj)

def memoized(f):
    """
    Wraps the _memoized decorator using functools so that,
    for example, the correct docstring will be used.
    """
    memoize = _memoized(f)

    @functools.wraps(f)
    def helper(*args, **kwargs):
        return memoize(*args, **kwargs)

    return helper

def extract_classpath(options):
    """
    Return the jvm classpath from a sequence of option strings.

    :param options: sequence of jvm options to search
    :returns: classpath in options
    :rtype: string
    """
    # a later classpath overwrites previous definitions so we have to search
    # from the end
    options = options[::-1]
    classpath = ''
    for i, x in enumerate(options):
        if x in ('-cp', '-classpath'):
            try:
                # cp is positioned before option because order is reversed,
                # take abs to avoid referencing from end
                cp_index = abs(i - 1)
                classpath = options[cp_index]
            except IndexError as e:
                classpath = ''
            break
    return classpath

def tree_pp(elem, level=0):
    """
    Pretty-prints an ElementTree.
    """
    i = "\n" + level * "  "
    if len(elem):
        if not elem.text or not elem.text.strip():
            elem.text = i + "  "
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
        for elem in elem:
            tree_pp(elem, level + 1)
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
    else:
        if level and (not elem.tail or not elem.tail.strip()):
            elem.tail = i

def dict2tree(elem, dictionary):
    for k, v in dictionary.items():
        if v:
            e = SubElement(elem, k)
            e.text = v

def dict2string(d, attribs=None):
    """
    PrettyPrints a dictionary as string.
    """
    s = []
    for k, v in d.items():
        if not attribs or k in attribs:
            if v:
                s.append("%s=%s" % (k, v))
    return ", ".join(s)

def sha1sum(filename, blocksize=65536):
    """
    Returns the sha1 hexdigest of a file.
    """
    hasher = hashlib.sha1()

    with open(filename, 'r') as afile:
        buf = afile.read(blocksize)
        while len(buf) > 0:
            hasher.update(buf)
            buf = afile.read(blocksize)

    return hasher.hexdigest()

@contextmanager
def tempdir(prefix='penchy-invocation', delete=False):
    """
    Execute in new created temporary directory.
    """
    fwd = os.getcwd()
    cwd = tempfile.mkdtemp(prefix=prefix)
    os.chdir(cwd)
    yield
    os.chdir(fwd)
    if delete:
        shutil.rmtree(cwd)

def find_bootstrap_client():
    """
    Returns the path of the penchy bootstrap client.
    """
    import penchy
    return pkg_resources.resource_filename('penchy',
            os.path.join('../', 'bin', 'penchy_bootstrap'))

def load_job(filename):
    """
    Loads a job.

    :param filename: filename of the job
    :type filename: string
    :param config: config file to export to job namespace
    """

    assert 'config' in sys.modules, "You have to load the penchyrc before a job"

    with disable_write_bytecode():
        job = imp.load_source('job', filename)
    log.info("Loaded job from %s" % filename)
    return job

def load_config(filename):
    """
    Loads the config module from filename. Looks
    in the current working directory as well.

    :param filename: filename of the config file
    :type filename: string
    :returns: config object
    """
    try:
        with disable_write_bytecode():
            config = imp.load_source('config', filename)
    except IOError:
        raise IOError("Config file could not be loaded from: %s" % filename)

    log.info("Loaded configuration from %s" % filename)
    return config

@contextmanager
def disable_write_bytecode():
    """
    Temporarily disables writing bytecode while executing.
    """
    old_state = sys.dont_write_bytecode
    sys.dont_write_bytecode = True
    yield
    sys.dont_write_bytecode = old_state
<|endoftext|>


<commit_before>class DistrictType(object):
    """
    Constants for what district an Event is a part of
    """
    # Event is not a district event
    NO_DISTRICT = 0

    MICHIGAN = 1
    MID_ATLANTIC = 2
    NEW_ENGLAND = 3
    PACIFIC_NORTHWEST = 4
    INDIANA = 5

    # Used for rendering
    type_names = {
        NO_DISTRICT: None,
        MICHIGAN: 'Michigan',
        MID_ATLANTIC: 'Mid Atlantic',
        NEW_ENGLAND: 'New England',
        PACIFIC_NORTHWEST: 'Pacific Northwest',
        INDIANA: 'Indiana',
    }

    type_abbrevs = {
        NO_DISTRICT: None,
        MICHIGAN: 'fim',
        MID_ATLANTIC: 'mar',
        NEW_ENGLAND: 'ne',
        PACIFIC_NORTHWEST: 'pnw',
        INDIANA: 'in',
    }

    # Names used on the FIRST website
    names = {
        'FIRST in Michigan': MICHIGAN,
        'Mid-Atlantic Robotics': MID_ATLANTIC,
        'New England': NEW_ENGLAND,
        'Pacific Northwest': PACIFIC_NORTHWEST,
        'Indiana': INDIANA,
    }

    abbrevs = {
        'fim': MICHIGAN,
        'mar': MID_ATLANTIC,
        'ne': NEW_ENGLAND,
        'pnw': PACIFIC_NORTHWEST,
        'in': INDIANA,
    }
<commit_msg>Update Indiana district name<commit_after>class DistrictType(object):
    """
    Constants for what district an Event is a part of
    """
    # Event is not a district event
    NO_DISTRICT = 0

    MICHIGAN = 1
    MID_ATLANTIC = 2
    NEW_ENGLAND = 3
    PACIFIC_NORTHWEST = 4
    INDIANA = 5

    # Used for rendering
    type_names = {
        NO_DISTRICT: None,
        MICHIGAN: 'Michigan',
        MID_ATLANTIC: 'Mid Atlantic',
        NEW_ENGLAND: 'New England',
        PACIFIC_NORTHWEST: 'Pacific Northwest',
        INDIANA: 'Indiana',
    }

    type_abbrevs = {
        NO_DISTRICT: None,
        MICHIGAN: 'fim',
        MID_ATLANTIC: 'mar',
        NEW_ENGLAND: 'ne',
        PACIFIC_NORTHWEST: 'pnw',
        INDIANA: 'in',
    }

    # Names used on the FIRST website
    names = {
        'FIRST in Michigan': MICHIGAN,
        'Mid-Atlantic Robotics': MID_ATLANTIC,
        'New England': NEW_ENGLAND,
        'Pacific Northwest': PACIFIC_NORTHWEST,
        'IndianaFIRST': INDIANA,
    }

    abbrevs = {
        'fim': MICHIGAN,
        'mar': MID_ATLANTIC,
        'ne': NEW_ENGLAND,
        'pnw': PACIFIC_NORTHWEST,
        'in': INDIANA,
    }
<|endoftext|>