<commit_before>import logging
import os
import sys
import time
import re
import shutil
import itertools

from pylons import config

from ckan.lib.cli import CkanCommand

REQUESTS_HEADER = {'content-type': 'application/json'}

class Archiver(CkanCommand):
    '''
    Download and save copies of all package resources.

    The result of each download attempt is saved to the CKAN task_status table, so the
    information can be used later for QA analysis.

    Usage:

        paster archiver init
           - Creates the database table archiver needs to run

        paster archiver update [{package-name/id}|{group-name/id}]
           - Archive all resources or just those belonging to a specific
             package or group, if specified

        paster archiver clean-status
           - Cleans the TaskStatus records that contain the status of each
             archived resource, whether it was successful or not, with errors.
             It does not change the cache_url etc. in the Resource

        paster archiver clean-cached-resources
           - Removes all cache_urls and other references to resource files on disk.

        paster archiver view [{dataset name/id}]
           - Views info archival info, in general and if you specify one, about
             a particular dataset\'s resources.

        paster archiver report [outputfile]
           - Generates a report on orphans, either resources where the path
             does not exist, or files on disk that don't have a corresponding
             orphan. The outputfile parameter is the name of the CSV output
             from running the report

        paster archiver delete-orphans [outputfile]
           - Deletes orphans that are files on disk with no corresponding
             resource. This uses the report command and will write out a
             report to [outputfile]

        paster archiver migrate-archive-dirs
           - Migrate the layout of the archived resource directories.
             Previous versions of ckanext-archiver stored resources on disk
             at: {resource-id}/filename.csv and this version puts them at:
             {2-chars-of-resource-id}/{resource-id}/filename.csv
             Running this moves them to the new locations and updates the
             cache_url on each resource to reflect the new location.
    '''
    # TODO
    #    paster archiver clean-files
    #       - Remove all archived resources

    summary = __doc__.split('\n')[0]
    usage = __doc__
    min_args = 0
    max_args = 2

    def __init__(self, name):
        super(Archiver, self).__init__(name)
        self.parser.add_option('-q', '--queue',
                               action='store',
                               dest='queue',
                               help='Send to a particular queue')

    def command(self):
        """
        Parse command line arguments and call appropriate method.
        """
        if not self.args or self.args[0] in ['--help', '-h', 'help']:
            print self.usage
            sys.exit(1)

        cmd = self.args[0]
        self._load_config()

        # Initialise logger after the config is loaded, so it is not disabled.
        self.log = logging.getLogger(__name__)

        if cmd == 'update':
            self.update()
        elif cmd == 'clean-status':
            self.clean_status()
        elif cmd == 'clean-cached-resources':
            self.clean_cached_resources()
        elif cmd == 'view':
            if len(self.args) == 2:
                self.view(self.args[1])
            else:
                self.view()
        elif cmd == 'report':
            if len(self.args) != 2:
                self.log.error('Command requires a parameter, the name of the output')
                return
            self.report(self.args[1], delete=False)
        elif cmd == 'delete-orphans':
            if len(self.args) != 2:
                self.log.error('Command requires a parameter, the name of the output')
                return
            self.report(self.args[1], delete=True)
        elif cmd == 'init':
            import ckan.model as model
            from ckanext.archiver.model import init_tables
            init_tables(model.meta.engine)
            self.log.info('Archiver tables are initialized')
        elif cmd == 'migrate-archive-dirs':
            self.migrate_archive_dirs()
        else:
            self.log.error('Command %s not recognized' % (cmd,))

    def update(self):
        from ckan import model
        from ckanext.archiver import plugin
        packages = []
        resources = []
        if len(self.args) > 1:
            for arg in self.args[1:]:
                # try arg as a group id/name
                group = model.Group.get(arg)
                if group:
                    packages.extend(group.packages())
                    if not self.options.queue:
                        self.options.queue = 'bulk'
                    continue
                # try arg as a package id/name
                pkg = model.Package.get(arg)
                if pkg:
                    packages.append(pkg)
                    if not self.options.queue:
                        self.options.queue = 'priority'
                    continue
                # try arg as a resource id
                res = model.Resource.get(arg)
                if res:
                    resources.append(res)
                    if not self.options.queue:
                        self.options.queue = 'priority'
                    continue
                else:
                    self.log.error('Could not recognize as a group, package '
                                   'or resource: %r', arg)
                    sys.exit(1)
        else:
            # all packages
            pkgs = model.Session.query(model.Package)\
                        .filter_by(state='active')\
                        .order_by('name').all()
            packages.extend(pkgs)
            if not self.options.queue:
                self.options.queue = 'bulk'

        if packages:
            self.log.info('Datasets to archive: %d', len(packages))
        if resources:
            self.log.info('Resources to archive: %d', len(resources))
        if not (packages or resources):
            self.log.error('No datasets or resources to process')
            sys.exit(1)

        self.log.info('Queue: %s', self.options.queue)
        for package in packages:
            pkg_resources = \
                [res for res in
                    itertools.chain.from_iterable(
                        (rg.resources_all for rg in package.resource_groups_all)
                    )
                 if res.state == 'active']
            self.log.info('Queuing dataset %s (%s resources)',
                          package.name, len(pkg_resources))
            for resource in pkg_resources:
                plugin.create_archiver_task(resource, self.options.queue)
            time.sleep(0.1)  # to try to avoid Redis getting overloaded

        for resource in resources:
            package = resource.resource_group.package
            self.log.info('Queuing resource %s/%s', package.name, resource.id)
            plugin.create_archiver_task(resource, self.options.queue)
            time.sleep(0.05)  # to try to avoid Redis getting overloaded

        self.log.info('Completed queueing')

    def view(self, package_ref=None):
        from ckan import model
        from ckanext.archiver.model import Archival

        r_q = model.Session.query(model.Resource).filter_by(state='active')
        print 'Resources: %i total' % r_q.count()
        a_q = model.Session.query(Archival)
        print 'Archived resources: %i total' % a_q.count()
        num_with_cache_url = a_q.filter(Archival.cache_url!='').count()
        print '                    %i with cache_url' % num_with_cache_url
        last_updated_res = a_q.order_by(Archival.updated.desc()).first()
        print 'Latest archival: %s' % (last_updated_res.updated.strftime('%Y-%m-%d %H:%M') if last_updated_res else '(no)')

        if package_ref:
            pkg = model.Package.get(package_ref)
            print 'Package %s %s' % (pkg.name, pkg.id)
            for res in pkg.resources:
                print 'Resource %s' % res.id
                for archival in a_q.filter_by(resource_id=res.id):
                    print '* %r' % archival

    def clean_status(self):
        from ckan import model
        from ckanext.archiver.model import Archival

        print 'Before:'
        self.view()

        q = model.Session.query(Archival)
        q.delete()
        model.Session.commit()

        print 'After:'
        self.view()

    def clean_cached_resources(self):
        from ckan import model
        from ckanext.archiver.model import Archival

        print 'Before:'
        self.view()

        q = model.Session.query(Archival).filter(Archival.cache_url != '')
        archivals = q.all()
        num_archivals = len(archivals)
        progress = 0
        for archival in archivals:
            archival.cache_url = None
            archival.cache_filepath = None
            archival.size = None
            archival.mimetype = None
            archival.hash = None
            progress += 1
            if progress % 1000 == 0:
                print 'Done %i/%i' % (progress, num_archivals)
                model.Session.commit()
        model.Session.commit()
        model.Session.remove()

        print 'After:'
        self.view()

    def report(self, output_file, delete=False):
        """
        Generates a report containing orphans (either files or resources)
        """
        import csv
        from ckan import model

        archive_root = config.get('ckanext-archiver.archive_dir')
        if not archive_root:
            log.error("Could not find archiver root")
            return

        # We'll use this to match the UUID part of the path
        uuid_re = re.compile(".*([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}).*")

        not_cached_active = 0
        not_cached_deleted = 0
        file_not_found_active = 0
        file_not_found_deleted = 0
        perm_error = 0
        file_no_resource = 0

        with open(output_file, "w") as f:
            writer = csv.writer(f)
            writer.writerow(["Resource ID", "Filepath", "Problem"])
            resources = {}
            for resource in model.Session.query(model.Resource).all():
                resources[resource.id] = True

                # Check the resource's cached_filepath
                fp = resource.extras.get('cache_filepath')
                if fp is None:
                    if resource.state == 'active':
                        not_cached_active += 1
                    else:
                        not_cached_deleted += 1
                    writer.writerow([resource.id, str(resource.extras), "Resource not cached: {0}".format(resource.state)])
                    continue

                # Check that the cached file is there and readable
                if not os.path.exists(fp):
                    if resource.state == 'active':
                        file_not_found_active += 1
                    else:
                        file_not_found_deleted += 1

                    writer.writerow([resource.id, fp.encode('utf-8'), "File not found: {0}".format(resource.state)])
                    continue

                try:
                    s = os.stat(fp)
                except OSError:
                    perm_error += 1
                    writer.writerow([resource.id, fp.encode('utf-8'), "File not readable"])
                    continue

            # Iterate over the archive root and check each file by matching the
            # resource_id part of the path to the resources dict
            for root, _, files in os.walk(archive_root):
                for filename in files:
                    archived_path = os.path.join(root, filename)
                    m = uuid_re.match(archived_path)
                    if not m:
                        writer.writerow([resource.id, archived_path, "Malformed path (no UUID)"])
                        continue

                    if not resources.get(m.groups(0)[0].strip(), False):
                        file_no_resource += 1

                        if delete:
                            try:
                                os.unlink(archived_path)
                                self.log.info("Unlinked {0}".format(archived_path))
                                os.rmdir(root)
                                self.log.info("Unlinked {0}".format(root))
                                writer.writerow([m.groups(0)[0], archived_path, "Resource not found, file deleted"])
                            except Exception, e:
                                self.log.error("Failed to unlink {0}: {1}".format(archived_path,e))
                        else:
                            writer.writerow([m.groups(0)[0], archived_path, "Resource not found"])

                        continue

        print "General info:"
        print "  Permission error reading file: {0}".format(perm_error)
        print "  file on disk but no resource: {0}".format(file_no_resource)
        print "  Total resources: {0}".format(model.Session.query(model.Resource).count())
        print "Active resource info:"
        print "  No cache_filepath: {0}".format(not_cached_active)
        print "  cache_filepath not on disk: {0}".format(file_not_found_active)
        print "Deleted resource info:"
        print "  No cache_filepath: {0}".format(not_cached_deleted)
        print "  cache_filepath not on disk: {0}".format(file_not_found_deleted)

    def migrate_archive_dirs(self):
        from ckan import model
        from ckan.logic import get_action

        site_user = get_action('get_site_user')(
            {'model': model, 'ignore_auth': True, 'defer_commit': True}, {}
        )

        site_url_base = config['ckan.cache_url_root'].rstrip('/')
        old_dir_regex = re.compile(r'(.*)/([a-f0-9\-]+)/([^/]*)$')
        new_dir_regex = re.compile(r'(.*)/[a-f0-9]{2}/[a-f0-9\-]{36}/[^/]*$')
        for resource in model.Session.query(model.Resource).\
            filter(model.Resource.state != model.State.DELETED):
            if not resource.cache_url or resource.cache_url == 'None':
                continue
            if new_dir_regex.match(resource.cache_url):
                print 'Resource with new url already: %s' % resource.cache_url
                continue
            match = old_dir_regex.match(resource.cache_url)
            if not match:
                print 'ERROR Could not match url: %s' % resource.cache_url
                continue
            url_base, res_id, filename = match.groups()
            # check the package isn't deleted
            # Need to refresh the resource's session
            resource = model.Session.query(model.Resource).get(resource.id)
            if resource.resource_group and resource.resource_group.package:
                if resource.resource_group.package.state == model.State.DELETED:
                    print 'Package is deleted'
                    continue

            if url_base != site_url_base:
                print 'ERROR Base URL is incorrect: %r != %r' % (url_base, site_url_base)
                continue

            # move the file
            filepath_base = config['ckanext-archiver.archive_dir']
            old_path = os.path.join(filepath_base, resource.id)
            new_dir = os.path.join(filepath_base, resource.id[:2])
            new_path = os.path.join(filepath_base, resource.id[:2], resource.id)
            new_filepath = os.path.join(new_path, filename)
            if not os.path.exists(new_dir):
                os.mkdir(new_dir)
            if os.path.exists(new_path) and not os.path.exists(old_path):
                print 'File already moved: %s' % new_path
            else:
                print 'File: "%s" -> "%s"' % (old_path, new_path)
                try:
                    shutil.move(old_path, new_path)
                except IOError, e:
                    print 'ERROR moving resource: %s' % e
                    continue

            # change the cache_url and cache_filepath
            new_cache_url = '/'.join((url_base, res_id[:2], res_id, filename))
            print 'cache_filepath: "%s" -> "%s"' % (resource.extras.get('cache_filepath'), new_filepath)
            print 'cache_url: "%s" -> "%s"' % (resource.cache_url, new_cache_url)
            context = {'model': model, 'user': site_user['name'], 'ignore_auth': True, 'session': model.Session}
            data_dict = {'id': resource.id}
            res_dict = get_action('resource_show')(context, data_dict)
            res_dict['cache_filepath'] = new_filepath
            res_dict['cache_url'] = new_cache_url
            data_dict = res_dict
            result = get_action('resource_update')(context, data_dict)
            if result.get('id') == res_id:
                print 'Successfully updated resource'
            else:
                print 'ERROR updating resource: %r' % result
<commit_msg>Fix for difficulty archiving "private" datasets in an group/organization. Still need to solve https://github.com/ckan/ckan/issues/1975 for groups.<commit_after>import logging
import os
import sys
import time
import re
import shutil
import itertools

from pylons import config

from ckan.lib.cli import CkanCommand

REQUESTS_HEADER = {'content-type': 'application/json'}

class Archiver(CkanCommand):
    '''
    Download and save copies of all package resources.

    The result of each download attempt is saved to the CKAN task_status table, so the
    information can be used later for QA analysis.

    Usage:

        paster archiver init
           - Creates the database table archiver needs to run

        paster archiver update [{package-name/id}|{group-name/id}]
           - Archive all resources or just those belonging to a specific
             package or group, if specified

        paster archiver clean-status
           - Cleans the TaskStatus records that contain the status of each
             archived resource, whether it was successful or not, with errors.
             It does not change the cache_url etc. in the Resource

        paster archiver clean-cached-resources
           - Removes all cache_urls and other references to resource files on disk.

        paster archiver view [{dataset name/id}]
           - Views info archival info, in general and if you specify one, about
             a particular dataset\'s resources.

        paster archiver report [outputfile]
           - Generates a report on orphans, either resources where the path
             does not exist, or files on disk that don't have a corresponding
             orphan. The outputfile parameter is the name of the CSV output
             from running the report

        paster archiver delete-orphans [outputfile]
           - Deletes orphans that are files on disk with no corresponding
             resource. This uses the report command and will write out a
             report to [outputfile]

        paster archiver migrate-archive-dirs
           - Migrate the layout of the archived resource directories.
             Previous versions of ckanext-archiver stored resources on disk
             at: {resource-id}/filename.csv and this version puts them at:
             {2-chars-of-resource-id}/{resource-id}/filename.csv
             Running this moves them to the new locations and updates the
             cache_url on each resource to reflect the new location.
    '''
    # TODO
    #    paster archiver clean-files
    #       - Remove all archived resources

    summary = __doc__.split('\n')[0]
    usage = __doc__
    min_args = 0
    max_args = 2

    def __init__(self, name):
        super(Archiver, self).__init__(name)
        self.parser.add_option('-q', '--queue',
                               action='store',
                               dest='queue',
                               help='Send to a particular queue')

    def command(self):
        """
        Parse command line arguments and call appropriate method.
        """
        if not self.args or self.args[0] in ['--help', '-h', 'help']:
            print self.usage
            sys.exit(1)

        cmd = self.args[0]
        self._load_config()

        # Initialise logger after the config is loaded, so it is not disabled.
        self.log = logging.getLogger(__name__)

        if cmd == 'update':
            self.update()
        elif cmd == 'clean-status':
            self.clean_status()
        elif cmd == 'clean-cached-resources':
            self.clean_cached_resources()
        elif cmd == 'view':
            if len(self.args) == 2:
                self.view(self.args[1])
            else:
                self.view()
        elif cmd == 'report':
            if len(self.args) != 2:
                self.log.error('Command requires a parameter, the name of the output')
                return
            self.report(self.args[1], delete=False)
        elif cmd == 'delete-orphans':
            if len(self.args) != 2:
                self.log.error('Command requires a parameter, the name of the output')
                return
            self.report(self.args[1], delete=True)
        elif cmd == 'init':
            import ckan.model as model
            from ckanext.archiver.model import init_tables
            init_tables(model.meta.engine)
            self.log.info('Archiver tables are initialized')
        elif cmd == 'migrate-archive-dirs':
            self.migrate_archive_dirs()
        else:
            self.log.error('Command %s not recognized' % (cmd,))

    def update(self):
        from ckan import model
        from ckanext.archiver import plugin
        packages = []
        resources = []
        if len(self.args) > 1:
            for arg in self.args[1:]:
                # try arg as a group id/name
                group = model.Group.get(arg)
                if group:
                    if group.is_organization:
                        packages.extend(
                            model.Session.query(model.Package)\
                                 .filter_by(owner_org=group.id))
                    else:
                        packages.extend(group.packages(with_private=True))
                    if not self.options.queue:
                        self.options.queue = 'bulk'
                    continue
                # try arg as a package id/name
                pkg = model.Package.get(arg)
                if pkg:
                    packages.append(pkg)
                    if not self.options.queue:
                        self.options.queue = 'priority'
                    continue
                # try arg as a resource id
                res = model.Resource.get(arg)
                if res:
                    resources.append(res)
                    if not self.options.queue:
                        self.options.queue = 'priority'
                    continue
                else:
                    self.log.error('Could not recognize as a group, package '
                                   'or resource: %r', arg)
                    sys.exit(1)
        else:
            # all packages
            pkgs = model.Session.query(model.Package)\
                        .filter_by(state='active')\
                        .order_by('name').all()
            packages.extend(pkgs)
            if not self.options.queue:
                self.options.queue = 'bulk'

        if packages:
            self.log.info('Datasets to archive: %d', len(packages))
        if resources:
            self.log.info('Resources to archive: %d', len(resources))
        if not (packages or resources):
            self.log.error('No datasets or resources to process')
            sys.exit(1)

        self.log.info('Queue: %s', self.options.queue)
        for package in packages:
            pkg_resources = \
                [res for res in
                    itertools.chain.from_iterable(
                        (rg.resources_all for rg in package.resource_groups_all)
                    )
                 if res.state == 'active']
            self.log.info('Queuing dataset %s (%s resources)',
                          package.name, len(pkg_resources))
            for resource in pkg_resources:
                plugin.create_archiver_task(resource, self.options.queue)
            time.sleep(0.1)  # to try to avoid Redis getting overloaded

        for resource in resources:
            package = resource.resource_group.package
            self.log.info('Queuing resource %s/%s', package.name, resource.id)
            plugin.create_archiver_task(resource, self.options.queue)
            time.sleep(0.05)  # to try to avoid Redis getting overloaded

        self.log.info('Completed queueing')

    def view(self, package_ref=None):
        from ckan import model
        from ckanext.archiver.model import Archival

        r_q = model.Session.query(model.Resource).filter_by(state='active')
        print 'Resources: %i total' % r_q.count()
        a_q = model.Session.query(Archival)
        print 'Archived resources: %i total' % a_q.count()
        num_with_cache_url = a_q.filter(Archival.cache_url!='').count()
        print '                    %i with cache_url' % num_with_cache_url
        last_updated_res = a_q.order_by(Archival.updated.desc()).first()
        print 'Latest archival: %s' % (last_updated_res.updated.strftime('%Y-%m-%d %H:%M') if last_updated_res else '(no)')

        if package_ref:
            pkg = model.Package.get(package_ref)
            print 'Package %s %s' % (pkg.name, pkg.id)
            for res in pkg.resources:
                print 'Resource %s' % res.id
                for archival in a_q.filter_by(resource_id=res.id):
                    print '* %r' % archival

    def clean_status(self):
        from ckan import model
        from ckanext.archiver.model import Archival

        print 'Before:'
        self.view()

        q = model.Session.query(Archival)
        q.delete()
        model.Session.commit()

        print 'After:'
        self.view()

    def clean_cached_resources(self):
        from ckan import model
        from ckanext.archiver.model import Archival

        print 'Before:'
        self.view()

        q = model.Session.query(Archival).filter(Archival.cache_url != '')
        archivals = q.all()
        num_archivals = len(archivals)
        progress = 0
        for archival in archivals:
            archival.cache_url = None
            archival.cache_filepath = None
            archival.size = None
            archival.mimetype = None
            archival.hash = None
            progress += 1
            if progress % 1000 == 0:
                print 'Done %i/%i' % (progress, num_archivals)
                model.Session.commit()
        model.Session.commit()
        model.Session.remove()

        print 'After:'
        self.view()

    def report(self, output_file, delete=False):
        """
        Generates a report containing orphans (either files or resources)
        """
        import csv
        from ckan import model

        archive_root = config.get('ckanext-archiver.archive_dir')
        if not archive_root:
            log.error("Could not find archiver root")
            return

        # We'll use this to match the UUID part of the path
        uuid_re = re.compile(".*([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}).*")

        not_cached_active = 0
        not_cached_deleted = 0
        file_not_found_active = 0
        file_not_found_deleted = 0
        perm_error = 0
        file_no_resource = 0

        with open(output_file, "w") as f:
            writer = csv.writer(f)
            writer.writerow(["Resource ID", "Filepath", "Problem"])
            resources = {}
            for resource in model.Session.query(model.Resource).all():
                resources[resource.id] = True

                # Check the resource's cached_filepath
                fp = resource.extras.get('cache_filepath')
                if fp is None:
                    if resource.state == 'active':
                        not_cached_active += 1
                    else:
                        not_cached_deleted += 1
                    writer.writerow([resource.id, str(resource.extras), "Resource not cached: {0}".format(resource.state)])
                    continue

                # Check that the cached file is there and readable
                if not os.path.exists(fp):
                    if resource.state == 'active':
                        file_not_found_active += 1
                    else:
                        file_not_found_deleted += 1

                    writer.writerow([resource.id, fp.encode('utf-8'), "File not found: {0}".format(resource.state)])
                    continue

                try:
                    s = os.stat(fp)
                except OSError:
                    perm_error += 1
                    writer.writerow([resource.id, fp.encode('utf-8'), "File not readable"])
                    continue

            # Iterate over the archive root and check each file by matching the
            # resource_id part of the path to the resources dict
            for root, _, files in os.walk(archive_root):
                for filename in files:
                    archived_path = os.path.join(root, filename)
                    m = uuid_re.match(archived_path)
                    if not m:
                        writer.writerow([resource.id, archived_path, "Malformed path (no UUID)"])
                        continue

                    if not resources.get(m.groups(0)[0].strip(), False):
                        file_no_resource += 1

                        if delete:
                            try:
                                os.unlink(archived_path)
                                self.log.info("Unlinked {0}".format(archived_path))
                                os.rmdir(root)
                                self.log.info("Unlinked {0}".format(root))
                                writer.writerow([m.groups(0)[0], archived_path, "Resource not found, file deleted"])
                            except Exception, e:
                                self.log.error("Failed to unlink {0}: {1}".format(archived_path,e))
                        else:
                            writer.writerow([m.groups(0)[0], archived_path, "Resource not found"])

                        continue

        print "General info:"
        print "  Permission error reading file: {0}".format(perm_error)
        print "  file on disk but no resource: {0}".format(file_no_resource)
        print "  Total resources: {0}".format(model.Session.query(model.Resource).count())
        print "Active resource info:"
        print "  No cache_filepath: {0}".format(not_cached_active)
        print "  cache_filepath not on disk: {0}".format(file_not_found_active)
        print "Deleted resource info:"
        print "  No cache_filepath: {0}".format(not_cached_deleted)
        print "  cache_filepath not on disk: {0}".format(file_not_found_deleted)

    def migrate_archive_dirs(self):
        from ckan import model
        from ckan.logic import get_action

        site_user = get_action('get_site_user')(
            {'model': model, 'ignore_auth': True, 'defer_commit': True}, {}
        )

        site_url_base = config['ckan.cache_url_root'].rstrip('/')
        old_dir_regex = re.compile(r'(.*)/([a-f0-9\-]+)/([^/]*)$')
        new_dir_regex = re.compile(r'(.*)/[a-f0-9]{2}/[a-f0-9\-]{36}/[^/]*$')
        for resource in model.Session.query(model.Resource).\
            filter(model.Resource.state != model.State.DELETED):
            if not resource.cache_url or resource.cache_url == 'None':
                continue
            if new_dir_regex.match(resource.cache_url):
                print 'Resource with new url already: %s' % resource.cache_url
                continue
            match = old_dir_regex.match(resource.cache_url)
            if not match:
                print 'ERROR Could not match url: %s' % resource.cache_url
                continue
            url_base, res_id, filename = match.groups()
            # check the package isn't deleted
            # Need to refresh the resource's session
            resource = model.Session.query(model.Resource).get(resource.id)
            if resource.resource_group and resource.resource_group.package:
                if resource.resource_group.package.state == model.State.DELETED:
                    print 'Package is deleted'
                    continue

            if url_base != site_url_base:
                print 'ERROR Base URL is incorrect: %r != %r' % (url_base, site_url_base)
                continue

            # move the file
            filepath_base = config['ckanext-archiver.archive_dir']
            old_path = os.path.join(filepath_base, resource.id)
            new_dir = os.path.join(filepath_base, resource.id[:2])
            new_path = os.path.join(filepath_base, resource.id[:2], resource.id)
            new_filepath = os.path.join(new_path, filename)
            if not os.path.exists(new_dir):
                os.mkdir(new_dir)
            if os.path.exists(new_path) and not os.path.exists(old_path):
                print 'File already moved: %s' % new_path
            else:
                print 'File: "%s" -> "%s"' % (old_path, new_path)
                try:
                    shutil.move(old_path, new_path)
                except IOError, e:
                    print 'ERROR moving resource: %s' % e
                    continue

            # change the cache_url and cache_filepath
            new_cache_url = '/'.join((url_base, res_id[:2], res_id, filename))
            print 'cache_filepath: "%s" -> "%s"' % (resource.extras.get('cache_filepath'), new_filepath)
            print 'cache_url: "%s" -> "%s"' % (resource.cache_url, new_cache_url)
            context = {'model': model, 'user': site_user['name'], 'ignore_auth': True, 'session': model.Session}
            data_dict = {'id': resource.id}
            res_dict = get_action('resource_show')(context, data_dict)
            res_dict['cache_filepath'] = new_filepath
            res_dict['cache_url'] = new_cache_url
            data_dict = res_dict
            result = get_action('resource_update')(context, data_dict)
            if result.get('id') == res_id:
                print 'Successfully updated resource'
            else:
                print 'ERROR updating resource: %r' % result
<|endoftext|>


<commit_before># coding=utf-8
import codecs
import os.path
from collections import namedtuple

import louis

from daisyproducer.dictionary.models import LocalWord
from django.utils.encoding import smart_unicode

TABLES_DIR = os.path.abspath("/usr/local/share/liblouis/tables")

GRADE1_TABLES = ['sbs-wordsplit.dis', 'sbs-de-core6.cti', 'sbs-de-accents.cti',
                 'sbs-special.cti', 'sbs-whitespace.mod', 'sbs-de-letsign.mod',
                 'sbs-numsign.mod', 'sbs-litdigit-upper.mod', 'sbs-de-core.mod',
                 'sbs-de-g1-core.mod', 'sbs-special.mod']

GRADE2_TABLES = GRADE1_TABLES[:]
GRADE2_TABLES[8:10] = ('sbs-de-g2-core.mod',)

GRADE2_TABLES_NAME = GRADE2_TABLES[:]
GRADE2_TABLES_NAME[8:10] = ('sbs-de-g2-name.mod',)

GRADE2_TABLES_PLACE = GRADE2_TABLES[:]
GRADE2_TABLES_PLACE[8:10] = ('sbs-de-g2-place.mod', 'sbs-de-g2-name.mod')

def getTables(grade, name=False, place=False):
    if grade == 1:
        return GRADE1_TABLES
    else:
        if place:
            return GRADE2_TABLES_PLACE
        elif name:
            return GRADE2_TABLES_NAME
        else:
            return GRADE2_TABLES

asciiToDotsMap = {
    u'A': "1",
    u'B': "12",
    u'C': "14",
    u'D': "145",
    u'E': "15",
    u'F': "124",
    u'G': "1245",
    u'H': "125",
    u'I': "24",
    u'J': "245",
    u'K': "13",
    u'L': "123",
    u'M': "134",
    u'N': "1345",
    u'O': "135",
    u'P': "1234",
    u'Q': "12345",
    u'R': "1235",
    u'S': "234",
    u'T': "2345",
    u'U': "136",
    u'V': "1236",
    u'X': "1346",
    u'Y': "13456",
    u'Z': "1356",
    u'0': "346",
    u'1': "16",
    u'2': "126",
    u'3': "146",
    u'4': "1456",
    u'5': "156",
    u'6': "1246",
    u'7': "12456",
    u'8': "1256",
    u'9': "246",
    u'&': "12346",
    u'%': "123456",
    u'[': "12356",
    u'^': "2346",
    u']': "23456",
    u'W': "2456",
    u',': "2",
    u';': "23",
    u':': "25",
    u'/': "256",
    u'?': "26",
    u'+': "235",
    u'=': "2356",
    u'(': "236",
    u'*': "35",
    u')': "356",
    u'.': "3",
    u'\\': "34",
    u'@': "345",
    u'#': "3456",
    u'"': "4",
    u'!': "5",
    u'>': "45",
    u'$': "46",
    u'_': "456",
    u'<': "56",
    u'\'': "6",
    u'à': "123568",
    u'á': "168",
    u'â': "1678",
    u'ã': "34678",
    u'å': "345678",
    u'æ': "478",
    u'ç': "1234678",
    u'è': "23468",
    u'é': "1234568",
    u'ê': "12678",
    u'ë': "12468",
    u'ì': "348",
    u'í': "1468",
    u'î': "14678",
    u'ï': "124568",
    u'ð': "23458",
    u'ñ': "13458",
    u'ò': "3468",
    u'ó': "14568",
    u'ô': "145678",
    u'õ': "1358",
    u'ø': "24678",
    u'ù': "234568",
    u'ú': "1568",
    u'û': "15678",
    u'ý': "24568",
    u'þ': "12348",
    u'ÿ': "134568",
    u'œ': "246789",
    # P36 ohne nachfolgende Trennmarke "m" (für "ver" u.ä.)
    # Note: in the SBS braille tables this dot pattern is represented by
    # the ascii character 'v'. The reason for this inconsistency is that
    # transcribers want to have a '-' in the user interface. Because the
    # dictionary doesn't contain normal P36, we can savely use '-' instead
    # of 'v' in the database. This is more convenient than having a discordancy
    # between the user interface and the database.
    u'-': "36a",
    u'ā': "4-1",
    u'ă': "4-1",
    u'ą': "4-1",
    u'ć': "4-14",
    u'ĉ': "4-14",
    u'ċ': "4-14",
    u'č': "4-14",
    u'ď': "4-145",
    u'đ': "4-145",
    u'ē': "4-15",
    u'ė': "4-15",
    u'ę': "4-15",
    u'ğ': "4-1245",
    u'ģ': "4-1245",
    u'ĥ': "4-125",
    u'ħ': "4-125",
    u'ĩ': "4-24",
    u'ī': "4-24",
    u'į': "4-24",
    u'ı': "4-24",
    u'ĳ': "4-245",
    u'ĵ': "4-245",
    u'ķ': "4-13",
    u'ĺ': "4-123",
    u'ļ': "4-123",
    u'ľ': "4-123",
    u'ŀ': "4-123",
    u'ł': "4-123",
    u'ń': "4-1345",
    u'ņ': "4-1345",
    u'ň': "4-1345",
    u'ŋ': "4-1345",
    u'ō': "4-135",
    u'ŏ': "4-135",
    u'ő': "4-135",
    u'ŕ': "4-1235",
    u'ŗ': "4-1235",
    u'ř': "4-1235",
    u'ś': "4-234",
    u'ŝ': "4-234",
    u'ş': "4-234",
    u'š': "4-234",
    u'ţ': "4-2345",
    u'ť': "4-2345",
    u'ŧ': "4-2345",
    u'ũ': "4-136",
    u'ū': "4-136",
    u'ŭ': "4-136",
    u'ů': "4-136",
    u'ű': "4-136",
    u'ų': "4-136",
    u'ŵ': "4-2456",
    u'ŷ': "4-13456",
    u'ź': "4-1356",
    u'ż': "4-1356",
    u'ž': "4-1356",
    u'ǎ': "4-1",
    u'ẁ': "4-2456",
    u'ẃ': "4-2456",
    u'ẅ': "4-2456",
    u'ỳ': "4-13456",
    }

# register a special handler which translates unknown utf-8 into
# \xhhhh escape sequences so liblouis can understand them.
backslashreplace_handler = codecs.lookup_error('backslashreplace')
def liblouis_handler(error):
    (replacement, pos) = backslashreplace_handler(error)
    return (replacement.replace('\u','\\x'), pos)
codecs.register_error('liblouis', liblouis_handler)

def word2dots(word):
    dots = [asciiToDotsMap[c] for c in word]
    return '-'.join(dots)

def writeTable(fileName, words, translate):
    f = codecs.open(os.path.join(TABLES_DIR, fileName), "w", "utf-8", 'liblouis')
    for (untranslated, contracted) in words:
        if translate(smart_unicode(untranslated)) != smart_unicode(contracted):
            # FIXME do we need to translate unichr(0x250A)) back to '|'?
            f.write("word %s %s\n" % (smart_unicode(untranslated), word2dots(smart_unicode(contracted))))
    f.close()

def writeWhiteListTables(words):
    writeTable('sbs-de-g1-white.mod',
               ((smart_unicode(word.homograph_disambiguation).replace('|', unichr(0x250A)) if word.type == 5 else word.untranslated, word.braille)
                for word in words.filter(grade=1)),
               lambda word: louis.translateString(getTables(1), word))
    writeTable('sbs-de-g2-white.mod',
               ((smart_unicode(word.homograph_disambiguation).replace('|', unichr(0x250A)) if word.type == 5 else word.untranslated, word.braille)
                for word in words.filter(grade=2).filter(type__in=(0, 1, 3, 5))),
               lambda word: louis.translateString(getTables(2), word))
    writeTable('sbs-de-g2-name-white.mod', ((word.untranslated, word.braille) for word in words.filter(grade=2).filter(type__in=(1,2))),
               lambda word: louis.translateString(getTables(2, name=True), word))
    writeTable('sbs-de-g2-place-white.mod', ((word.untranslated, word.braille) for word in words.filter(grade=2).filter(type__in=(3,4))),
               lambda word: louis.translateString(getTables(2, place=True), word))

def writeLocalTables(changedDocuments):
    for document in changedDocuments:
        words = LocalWord.objects.filter(document=document).order_by('untranslated')
        writeTable('sbs-de-g1-white-%s.mod' % document.identifier,
                   ((smart_unicode(word.homograph_disambiguation).replace('|', unichr(0x250A)) if word.type == 5 else word.untranslated, word.braille)
                    for word in words.filter(grade=1)),
                   lambda word: louis.translateString(getTables(1), word))
        writeTable('sbs-de-g2-white-%s.mod' % document.identifier,
                   ((smart_unicode(word.homograph_disambiguation).replace('|', unichr(0x250A)) if word.type == 5 else word.untranslated, word.braille)
                    for word in words.filter(grade=2).filter(type__in=(0, 1, 3, 5))),
                   lambda word: louis.translateString(getTables(2), word))
        writeTable('sbs-de-g2-name-white-%s.mod' % document.identifier,
                   ((word.untranslated, word.braille) for word in words.filter(grade=2).filter(type__in=(1,2))),
                   lambda word: louis.translateString(getTables(2, name=True), word))
        writeTable('sbs-de-g2-place-white-%s.mod' % document.identifier,
                   ((word.untranslated, word.braille) for word in words.filter(grade=2).filter(type__in=(3,4))),
                  lambda word: louis.translateString(getTables(2, place=True), word))

<commit_msg>Catch the exception when word2dots has no mapping<commit_after># coding=utf-8
import codecs
import os.path
import sys

from collections import namedtuple

import louis

from daisyproducer.dictionary.models import LocalWord
from django.utils.encoding import smart_unicode

TABLES_DIR = os.path.abspath("/usr/local/share/liblouis/tables")

GRADE1_TABLES = ['sbs-wordsplit.dis', 'sbs-de-core6.cti', 'sbs-de-accents.cti',
                 'sbs-special.cti', 'sbs-whitespace.mod', 'sbs-de-letsign.mod',
                 'sbs-numsign.mod', 'sbs-litdigit-upper.mod', 'sbs-de-core.mod',
                 'sbs-de-g1-core.mod', 'sbs-special.mod']

GRADE2_TABLES = GRADE1_TABLES[:]
GRADE2_TABLES[8:10] = ('sbs-de-g2-core.mod',)

GRADE2_TABLES_NAME = GRADE2_TABLES[:]
GRADE2_TABLES_NAME[8:10] = ('sbs-de-g2-name.mod',)

GRADE2_TABLES_PLACE = GRADE2_TABLES[:]
GRADE2_TABLES_PLACE[8:10] = ('sbs-de-g2-place.mod', 'sbs-de-g2-name.mod')

def getTables(grade, name=False, place=False):
    if grade == 1:
        return GRADE1_TABLES
    else:
        if place:
            return GRADE2_TABLES_PLACE
        elif name:
            return GRADE2_TABLES_NAME
        else:
            return GRADE2_TABLES

asciiToDotsMap = {
    u'A': "1",
    u'B': "12",
    u'C': "14",
    u'D': "145",
    u'E': "15",
    u'F': "124",
    u'G': "1245",
    u'H': "125",
    u'I': "24",
    u'J': "245",
    u'K': "13",
    u'L': "123",
    u'M': "134",
    u'N': "1345",
    u'O': "135",
    u'P': "1234",
    u'Q': "12345",
    u'R': "1235",
    u'S': "234",
    u'T': "2345",
    u'U': "136",
    u'V': "1236",
    u'X': "1346",
    u'Y': "13456",
    u'Z': "1356",
    u'0': "346",
    u'1': "16",
    u'2': "126",
    u'3': "146",
    u'4': "1456",
    u'5': "156",
    u'6': "1246",
    u'7': "12456",
    u'8': "1256",
    u'9': "246",
    u'&': "12346",
    u'%': "123456",
    u'[': "12356",
    u'^': "2346",
    u']': "23456",
    u'W': "2456",
    u',': "2",
    u';': "23",
    u':': "25",
    u'/': "256",
    u'?': "26",
    u'+': "235",
    u'=': "2356",
    u'(': "236",
    u'*': "35",
    u')': "356",
    u'.': "3",
    u'\\': "34",
    u'@': "345",
    u'#': "3456",
    u'"': "4",
    u'!': "5",
    u'>': "45",
    u'$': "46",
    u'_': "456",
    u'<': "56",
    u'\'': "6",
    u'à': "123568",
    u'á': "168",
    u'â': "1678",
    u'ã': "34678",
    u'å': "345678",
    u'æ': "478",
    u'ç': "1234678",
    u'è': "23468",
    u'é': "1234568",
    u'ê': "12678",
    u'ë': "12468",
    u'ì': "348",
    u'í': "1468",
    u'î': "14678",
    u'ï': "124568",
    u'ð': "23458",
    u'ñ': "13458",
    u'ò': "3468",
    u'ó': "14568",
    u'ô': "145678",
    u'õ': "1358",
    u'ø': "24678",
    u'ù': "234568",
    u'ú': "1568",
    u'û': "15678",
    u'ý': "24568",
    u'þ': "12348",
    u'ÿ': "134568",
    u'œ': "246789",
    # P36 ohne nachfolgende Trennmarke "m" (für "ver" u.ä.)
    # Note: in the SBS braille tables this dot pattern is represented by
    # the ascii character 'v'. The reason for this inconsistency is that
    # transcribers want to have a '-' in the user interface. Because the
    # dictionary doesn't contain normal P36, we can savely use '-' instead
    # of 'v' in the database. This is more convenient than having a discordancy
    # between the user interface and the database.
    u'-': "36a",
    u'ā': "4-1",
    u'ă': "4-1",
    u'ą': "4-1",
    u'ć': "4-14",
    u'ĉ': "4-14",
    u'ċ': "4-14",
    u'č': "4-14",
    u'ď': "4-145",
    u'đ': "4-145",
    u'ē': "4-15",
    u'ė': "4-15",
    u'ę': "4-15",
    u'ğ': "4-1245",
    u'ģ': "4-1245",
    u'ĥ': "4-125",
    u'ħ': "4-125",
    u'ĩ': "4-24",
    u'ī': "4-24",
    u'į': "4-24",
    u'ı': "4-24",
    u'ĳ': "4-245",
    u'ĵ': "4-245",
    u'ķ': "4-13",
    u'ĺ': "4-123",
    u'ļ': "4-123",
    u'ľ': "4-123",
    u'ŀ': "4-123",
    u'ł': "4-123",
    u'ń': "4-1345",
    u'ņ': "4-1345",
    u'ň': "4-1345",
    u'ŋ': "4-1345",
    u'ō': "4-135",
    u'ŏ': "4-135",
    u'ő': "4-135",
    u'ŕ': "4-1235",
    u'ŗ': "4-1235",
    u'ř': "4-1235",
    u'ś': "4-234",
    u'ŝ': "4-234",
    u'ş': "4-234",
    u'š': "4-234",
    u'ţ': "4-2345",
    u'ť': "4-2345",
    u'ŧ': "4-2345",
    u'ũ': "4-136",
    u'ū': "4-136",
    u'ŭ': "4-136",
    u'ů': "4-136",
    u'ű': "4-136",
    u'ų': "4-136",
    u'ŵ': "4-2456",
    u'ŷ': "4-13456",
    u'ź': "4-1356",
    u'ż': "4-1356",
    u'ž': "4-1356",
    u'ǎ': "4-1",
    u'ẁ': "4-2456",
    u'ẃ': "4-2456",
    u'ẅ': "4-2456",
    u'ỳ': "4-13456",
    }

# register a special handler which translates unknown utf-8 into
# \xhhhh escape sequences so liblouis can understand them.
backslashreplace_handler = codecs.lookup_error('backslashreplace')
def liblouis_handler(error):
    (replacement, pos) = backslashreplace_handler(error)
    return (replacement.replace('\u','\\x'), pos)
codecs.register_error('liblouis', liblouis_handler)

def word2dots(word):
    dots = [asciiToDotsMap[c] for c in word]
    return '-'.join(dots)

def writeTable(fileName, words, translate):
    f = codecs.open(os.path.join(TABLES_DIR, fileName), "w", "utf-8", 'liblouis')
    for (untranslated, contracted) in words:
        if translate(smart_unicode(untranslated)) != smart_unicode(contracted):
            # FIXME do we need to translate unichr(0x250A)) back to '|'?
            try:
                f.write("word %s %s\n" % (smart_unicode(untranslated), word2dots(smart_unicode(contracted))))
            except KeyError:
                sys.stderr.write("Error: unknown char in %s\n" % (contracted,))
    f.close()

def writeWhiteListTables(words):
    writeTable('sbs-de-g1-white.mod',
               ((smart_unicode(word.homograph_disambiguation).replace('|', unichr(0x250A)) if word.type == 5 else word.untranslated, word.braille)
                for word in words.filter(grade=1)),
               lambda word: louis.translateString(getTables(1), word))
    writeTable('sbs-de-g2-white.mod',
               ((smart_unicode(word.homograph_disambiguation).replace('|', unichr(0x250A)) if word.type == 5 else word.untranslated, word.braille)
                for word in words.filter(grade=2).filter(type__in=(0, 1, 3, 5))),
               lambda word: louis.translateString(getTables(2), word))
    writeTable('sbs-de-g2-name-white.mod', ((word.untranslated, word.braille) for word in words.filter(grade=2).filter(type__in=(1,2))),
               lambda word: louis.translateString(getTables(2, name=True), word))
    writeTable('sbs-de-g2-place-white.mod', ((word.untranslated, word.braille) for word in words.filter(grade=2).filter(type__in=(3,4))),
               lambda word: louis.translateString(getTables(2, place=True), word))

def writeLocalTables(changedDocuments):
    for document in changedDocuments:
        words = LocalWord.objects.filter(document=document).order_by('untranslated')
        writeTable('sbs-de-g1-white-%s.mod' % document.identifier,
                   ((smart_unicode(word.homograph_disambiguation).replace('|', unichr(0x250A)) if word.type == 5 else word.untranslated, word.braille)
                    for word in words.filter(grade=1)),
                   lambda word: louis.translateString(getTables(1), word))
        writeTable('sbs-de-g2-white-%s.mod' % document.identifier,
                   ((smart_unicode(word.homograph_disambiguation).replace('|', unichr(0x250A)) if word.type == 5 else word.untranslated, word.braille)
                    for word in words.filter(grade=2).filter(type__in=(0, 1, 3, 5))),
                   lambda word: louis.translateString(getTables(2), word))
        writeTable('sbs-de-g2-name-white-%s.mod' % document.identifier,
                   ((word.untranslated, word.braille) for word in words.filter(grade=2).filter(type__in=(1,2))),
                   lambda word: louis.translateString(getTables(2, name=True), word))
        writeTable('sbs-de-g2-place-white-%s.mod' % document.identifier,
                   ((word.untranslated, word.braille) for word in words.filter(grade=2).filter(type__in=(3,4))),
                  lambda word: louis.translateString(getTables(2, place=True), word))

<|endoftext|>


<commit_before>"""Show config player."""
from mpf.config_players.device_config_player import DeviceConfigPlayer

reserved_keys = ["show", "priority", "speed", "block_queue", "start_step", "loops", "sync_ms", "manual_advance",
                 "key", "show_tokens", "events_when_played", "events_when_stopped", "events_when_looped",
                 "events_when_paused", "events_when_resumed", "events_when_advanced",
                 "events_when_stepped_back", "events_when_updated", "events_when_completed"]

class ShowPlayer(DeviceConfigPlayer):

    """Plays, starts, stops, pauses, resumes or advances shows based on config."""

    config_file_section = 'show_player'
    show_section = 'shows'
    allow_placeholders_in_keys = True

    __slots__ = ["_actions"]

    def __init__(self, machine):
        """Initialise show player."""
        super().__init__(machine)
        self._actions = {
            'play': self._play,
            'stop': self._stop,
            'pause': self._pause,
            'resume': self._resume,
            'advance': self._advance,
            'step_back': self._step_back,
            'update': self._update
        }

    # pylint: disable-msg=too-many-arguments
    def play(self, settings, context, calling_context, priority=0, **kwargs):
        """Play, start, stop, pause, resume or advance show based on config."""
        # make sure all shows play in sync
        queue = kwargs.get("queue", None)
        start_time = kwargs.get("start_time", None)
        if not start_time:
            start_time = self.machine.clock.get_time()
        for show, show_settings in settings.items():
            # Look for a conditional event in the show name
            if show.condition and show.condition.evaluate(kwargs):
                continue

            show_settings = dict(show_settings)
            if 'hold' in show_settings and show_settings['hold'] is not None:
                raise AssertionError(
                    "Setting 'hold' is no longer supported for shows. Use duration -1 in your show.")
            try:
                show_settings['priority'] += priority
            except KeyError:
                show_settings['priority'] = priority
            # todo need to add this key back to the config player

            self._update_show(show.name, show_settings, context, queue, start_time, kwargs)

    def _expand_device(self, device):
        # parse conditionals
        devices = super()._expand_device(device)
        for index, device_entry in enumerate(devices):
            devices[index] = self.machine.placeholder_manager.parse_conditional_template(device_entry)
        return devices

    def _expand_device_config(self, device_settings):
        """Validate show_tokens."""
        for key in reserved_keys:
            if key in device_settings["show_tokens"]:
                self.raise_config_error("Key {} is not allowed in show_tokens of your show_player because it is also "
                                        "an option in show_player. Did indent that option too far?".format(key), 1)
        return device_settings

    def handle_subscription_change(self, value, settings, priority, context):
        """Handle subscriptions."""
        instance_dict = self._get_instance_dict(context)
        for show, show_settings in settings.items():
            show_settings = dict(show_settings)
            if show_settings['action'] != 'play':
                raise AssertionError("Can only use action play with subscriptions.")

            if 'key' in show_settings and show_settings['key']:
                key = show_settings['key']
            else:
                key = show.name

            if value:
                self._play(key, instance_dict, show.name, show_settings, False, None, {})
            else:
                self._stop(key, instance_dict, show.name, show_settings, False, None, {})

    # pylint: disable-msg=too-many-arguments
    def _play(self, key, instance_dict, show, show_settings, queue, start_time, placeholder_args):
        stop_callback = None
        if show_settings['block_queue']:
            if not queue:
                raise AssertionError("block_queue can only be used with a queue event.")
            queue.wait()
            stop_callback = queue.clear

        start_step = show_settings['start_step'].evaluate(placeholder_args)
        show_tokens = {k: v.evaluate(placeholder_args.get("show_tokens", {}))
                       for k, v in show_settings['show_tokens'].items()}

        show_config = self.machine.show_controller.create_show_config(
            show, show_settings['priority'], show_settings['speed'], show_settings['loops'], show_settings['sync_ms'],
            show_settings['manual_advance'], show_tokens, show_settings['events_when_played'],
            show_settings['events_when_stopped'], show_settings['events_when_looped'],
            show_settings['events_when_paused'], show_settings['events_when_resumed'],
            show_settings['events_when_advanced'], show_settings['events_when_stepped_back'],
            show_settings['events_when_updated'], show_settings['events_when_completed'])

        previous_show = instance_dict.get(key, None)

        instance_dict[key] = self.machine.show_controller.replace_or_advance_show(previous_show, show_config,
                                                                                start_step, start_time, stop_callback)

    @staticmethod
    def _stop(key, instance_dict, show, show_settings, queue, start_time, placeholder_args):
        del show
        del show_settings
        del queue
        del start_time
        del placeholder_args
        if key in instance_dict:
            instance_dict[key].stop()
            del instance_dict[key]

    @staticmethod
    def _pause(key, instance_dict, show, show_settings, queue, start_time, placeholder_args):
        del show
        del show_settings
        del queue
        del start_time
        del placeholder_args
        if key in instance_dict:
            instance_dict[key].pause()

    @staticmethod
    def _resume(key, instance_dict, show, show_settings, queue, start_time, placeholder_args):
        del show
        del show_settings
        del queue
        del start_time
        del placeholder_args
        if key in instance_dict:
            instance_dict[key].resume()

    @staticmethod
    def _advance(key, instance_dict, show, show_settings, queue, start_time, placeholder_args):
        del show
        del show_settings
        del queue
        del start_time
        del placeholder_args
        if key in instance_dict:
            instance_dict[key].advance()

    @staticmethod
    def _step_back(key, instance_dict, show, show_settings, queue, start_time, placeholder_args):
        del show
        del show_settings
        del queue
        del start_time
        del placeholder_args
        if key in instance_dict:
            instance_dict[key].step_back()

    @staticmethod
    def _update(key, instance_dict, show, show_settings, queue, start_time, placeholder_args):
        del show
        del queue
        del start_time
        del placeholder_args
        if key in instance_dict:
            instance_dict[key].update(
                show_tokens=show_settings['show_tokens'],
                priority=show_settings['priority'])

    # pylint: disable-msg=too-many-arguments
    def _update_show(self, show, show_settings, context, queue, start_time, placeholder_args):
        instance_dict = self._get_instance_dict(context)
        if 'key' in show_settings and show_settings['key']:
            key = show_settings['key']
        else:
            key = show

        try:
            action = self._actions[show_settings['action']]
        except KeyError:
            raise AssertionError("Invalid action {} in show_player {}".format(
                show_settings['action'], key))

        if 'show' in show_settings and show_settings['show']:
            show_name = show_settings['show']
        else:
            show_name = show

        action(key, instance_dict, show_name, show_settings, queue, start_time, placeholder_args)

    def clear_context(self, context):
        """Stop running shows from context."""
        for show in self._get_instance_dict(context).values():
            show.stop()
        self._reset_instance_dict(context)

    def get_express_config(self, value):
        """Parse express config."""
        return {"action": value}
<commit_msg>Fix inverted evaluation on show conditions<commit_after>"""Show config player."""
from mpf.config_players.device_config_player import DeviceConfigPlayer

reserved_keys = ["show", "priority", "speed", "block_queue", "start_step", "loops", "sync_ms", "manual_advance",
                 "key", "show_tokens", "events_when_played", "events_when_stopped", "events_when_looped",
                 "events_when_paused", "events_when_resumed", "events_when_advanced",
                 "events_when_stepped_back", "events_when_updated", "events_when_completed"]

class ShowPlayer(DeviceConfigPlayer):

    """Plays, starts, stops, pauses, resumes or advances shows based on config."""

    config_file_section = 'show_player'
    show_section = 'shows'
    allow_placeholders_in_keys = True

    __slots__ = ["_actions"]

    def __init__(self, machine):
        """Initialise show player."""
        super().__init__(machine)
        self._actions = {
            'play': self._play,
            'stop': self._stop,
            'pause': self._pause,
            'resume': self._resume,
            'advance': self._advance,
            'step_back': self._step_back,
            'update': self._update
        }

    # pylint: disable-msg=too-many-arguments
    def play(self, settings, context, calling_context, priority=0, **kwargs):
        """Play, start, stop, pause, resume or advance show based on config."""
        # make sure all shows play in sync
        queue = kwargs.get("queue", None)
        start_time = kwargs.get("start_time", None)
        if not start_time:
            start_time = self.machine.clock.get_time()
        for show, show_settings in settings.items():
            # Look for a conditional event in the show name
            if show.condition and not show.condition.evaluate(kwargs):
                continue

            show_settings = dict(show_settings)
            if 'hold' in show_settings and show_settings['hold'] is not None:
                raise AssertionError(
                    "Setting 'hold' is no longer supported for shows. Use duration -1 in your show.")
            try:
                show_settings['priority'] += priority
            except KeyError:
                show_settings['priority'] = priority
            # todo need to add this key back to the config player

            self._update_show(show.name, show_settings, context, queue, start_time, kwargs)

    def _expand_device(self, device):
        # parse conditionals
        devices = super()._expand_device(device)
        for index, device_entry in enumerate(devices):
            devices[index] = self.machine.placeholder_manager.parse_conditional_template(device_entry)
        return devices

    def _expand_device_config(self, device_settings):
        """Validate show_tokens."""
        for key in reserved_keys:
            if key in device_settings["show_tokens"]:
                self.raise_config_error("Key {} is not allowed in show_tokens of your show_player because it is also "
                                        "an option in show_player. Did indent that option too far?".format(key), 1)
        return device_settings

    def handle_subscription_change(self, value, settings, priority, context):
        """Handle subscriptions."""
        instance_dict = self._get_instance_dict(context)
        for show, show_settings in settings.items():
            show_settings = dict(show_settings)
            if show_settings['action'] != 'play':
                raise AssertionError("Can only use action play with subscriptions.")

            if 'key' in show_settings and show_settings['key']:
                key = show_settings['key']
            else:
                key = show.name

            if value:
                self._play(key, instance_dict, show.name, show_settings, False, None, {})
            else:
                self._stop(key, instance_dict, show.name, show_settings, False, None, {})

    # pylint: disable-msg=too-many-arguments
    def _play(self, key, instance_dict, show, show_settings, queue, start_time, placeholder_args):
        stop_callback = None
        if show_settings['block_queue']:
            if not queue:
                raise AssertionError("block_queue can only be used with a queue event.")
            queue.wait()
            stop_callback = queue.clear

        start_step = show_settings['start_step'].evaluate(placeholder_args)
        show_tokens = {k: v.evaluate(placeholder_args.get("show_tokens", {}))
                       for k, v in show_settings['show_tokens'].items()}

        show_config = self.machine.show_controller.create_show_config(
            show, show_settings['priority'], show_settings['speed'], show_settings['loops'], show_settings['sync_ms'],
            show_settings['manual_advance'], show_tokens, show_settings['events_when_played'],
            show_settings['events_when_stopped'], show_settings['events_when_looped'],
            show_settings['events_when_paused'], show_settings['events_when_resumed'],
            show_settings['events_when_advanced'], show_settings['events_when_stepped_back'],
            show_settings['events_when_updated'], show_settings['events_when_completed'])

        previous_show = instance_dict.get(key, None)

        instance_dict[key] = self.machine.show_controller.replace_or_advance_show(previous_show, show_config,
                                                                                start_step, start_time, stop_callback)

    @staticmethod
    def _stop(key, instance_dict, show, show_settings, queue, start_time, placeholder_args):
        del show
        del show_settings
        del queue
        del start_time
        del placeholder_args
        if key in instance_dict:
            instance_dict[key].stop()
            del instance_dict[key]

    @staticmethod
    def _pause(key, instance_dict, show, show_settings, queue, start_time, placeholder_args):
        del show
        del show_settings
        del queue
        del start_time
        del placeholder_args
        if key in instance_dict:
            instance_dict[key].pause()

    @staticmethod
    def _resume(key, instance_dict, show, show_settings, queue, start_time, placeholder_args):
        del show
        del show_settings
        del queue
        del start_time
        del placeholder_args
        if key in instance_dict:
            instance_dict[key].resume()

    @staticmethod
    def _advance(key, instance_dict, show, show_settings, queue, start_time, placeholder_args):
        del show
        del show_settings
        del queue
        del start_time
        del placeholder_args
        if key in instance_dict:
            instance_dict[key].advance()

    @staticmethod
    def _step_back(key, instance_dict, show, show_settings, queue, start_time, placeholder_args):
        del show
        del show_settings
        del queue
        del start_time
        del placeholder_args
        if key in instance_dict:
            instance_dict[key].step_back()

    @staticmethod
    def _update(key, instance_dict, show, show_settings, queue, start_time, placeholder_args):
        del show
        del queue
        del start_time
        del placeholder_args
        if key in instance_dict:
            instance_dict[key].update(
                show_tokens=show_settings['show_tokens'],
                priority=show_settings['priority'])

    # pylint: disable-msg=too-many-arguments
    def _update_show(self, show, show_settings, context, queue, start_time, placeholder_args):
        instance_dict = self._get_instance_dict(context)
        if 'key' in show_settings and show_settings['key']:
            key = show_settings['key']
        else:
            key = show

        try:
            action = self._actions[show_settings['action']]
        except KeyError:
            raise AssertionError("Invalid action {} in show_player {}".format(
                show_settings['action'], key))

        if 'show' in show_settings and show_settings['show']:
            show_name = show_settings['show']
        else:
            show_name = show

        action(key, instance_dict, show_name, show_settings, queue, start_time, placeholder_args)

    def clear_context(self, context):
        """Stop running shows from context."""
        for show in self._get_instance_dict(context).values():
            show.stop()
        self._reset_instance_dict(context)

    def get_express_config(self, value):
        """Parse express config."""
        return {"action": value}
<|endoftext|>


<commit_before>from msmbuilder.tpt import mfpt
import numpy as np

def test_create_perturb_params():
    # Test with a 10x10 counts matrix, with all entries in the counts set to 100
    countsmat = 100 * np.ones((10,10))
    params = mfpt.create_perturb_params(countsmat)
    # Check dimensions of outputs are equal to those of inputs
    for param in params:
        assert np.shape(param) == np.shape(countsmat)

def test_perturb_tmat():
    # The transition matrix is perturbed under the CLT approximation, which is only valid for well-sampled data w.r.t. transition probability (tprob >> 1 / row-summed counts)
    countsmat = 100 * np.ones((10,10)) # 10-state MSM, 1000 counts per state, 100 transition events between states, no zero entries
    params = mfpt.create_perturb_params(countsmat)
    new_transmat = (mfpt.perturb_tmat(params[0], params[1]))
    # All transition probabilities are by design nonzero, so there should be no nonzero entries after the perturbation
    assert len(np.where(new_transmat == 0)[0]) == 0
    # Now let's assume you have a poorly sampled dataset where all elements in the counts matrix is 1
    countsmat = np.ones((10,10))
    params = mfpt.create_perturb_params(countsmat)
    new_transmat = (mfpt.perturb_tmat(params[0], params[1]))
    # Your perturbed transition matrix will have several negative values (set automatically to 0), indicating this method probably isn't appropriate for your dataset
    # (This will also cause your distribution of MFPTs to have very obvious outliers to an otherwise approximately Gaussian distribution due to the artificial zeros in the transition matrix)
    assert len(np.where(new_transmat == 0)[0] > 0)

<commit_msg>Updated test<commit_after><|endoftext|>


<commit_before>###############################################################################
##
##  Copyright 2011 Tavendo GmbH
##
##  Licensed under the Apache License, Version 2.0 (the "License");
##  you may not use this file except in compliance with the License.
##  You may obtain a copy of the License at
##
##      http://www.apache.org/licenses/LICENSE-2.0
##
##  Unless required by applicable law or agreed to in writing, software
##  distributed under the License is distributed on an "AS IS" BASIS,
##  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
##  See the License for the specific language governing permissions and
##  limitations under the License.
##
###############################################################################

from websocket import WebSocketService, WebSocketServiceConnection, HttpException
import json

class FuzzingServiceConnection(WebSocketServiceConnection):

   def onConnect(self, host, uri, origin, protocols):
      #print "Client connected", host, uri, origin, protocols
      pass

   def onOpen(self):
      #print "Connection now open."
      pass

   def onClose(self):
      #print "Client lost."
      pass

   def onMessage(self, msg, binary):
      if not binary:
         obj = json.loads(msg)
         print obj
         if obj[0] == "sendframe":
            pl = obj[1]["payload"].encode("UTF-8")
            self.sendFrame(opcode = obj[1]["opcode"], payload = pl, fin = obj[1]["fin"])

class FuzzingService(WebSocketService):

   def __init__(self, debug = False):
      self.debug = debug
      self.protocol = FuzzingServiceConnection

   def startFactory(self):
      pass

   def stopFactory(self):
      pass
<commit_msg>more fuzzing tests<commit_after>###############################################################################
##
##  Copyright 2011 Tavendo GmbH
##
##  Licensed under the Apache License, Version 2.0 (the "License");
##  you may not use this file except in compliance with the License.
##  You may obtain a copy of the License at
##
##      http://www.apache.org/licenses/LICENSE-2.0
##
##  Unless required by applicable law or agreed to in writing, software
##  distributed under the License is distributed on an "AS IS" BASIS,
##  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
##  See the License for the specific language governing permissions and
##  limitations under the License.
##
###############################################################################

from websocket import WebSocketService, WebSocketServiceConnection, HttpException
import json

class FuzzingServiceConnection(WebSocketServiceConnection):

   def onPong(self, payload):
      if self.debug:
         #print self.peer, "PONG received", unicode(payload, "UTF-8")
         print self.peer, "PONG received", payload

   def onMessage(self, msg, binary):
      if not binary:
         obj = json.loads(msg)
         if obj[0] == "sendframe":
            pl = obj[1].get("payload", "")
            self.sendFrame(opcode = obj[1]["opcode"],
                           payload = pl.encode("UTF-8"),
                           fin = obj[1].get("fin", True),
                           rsv = obj[1].get("rsv", 0),
                           mask = obj[1].get("mask", None))

class FuzzingService(WebSocketService):

   def __init__(self, debug = False):
      self.debug = debug
      self.protocol = FuzzingServiceConnection

   def startFactory(self):
      pass

   def stopFactory(self):
      pass
<|endoftext|>


<commit_before># No shebang line, this module is meant to be imported
#
# This file is part of PyFarm.
# Copyright (C) 2008-2012 Oliver Palmer
#
# PyFarm is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyFarm is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with PyFarm.  If not, see <http://www.gnu.org/licenses/>.

import datetime
import sqlalchemy as sql

from pyfarm.preferences import prefs
from pyfarm.db.session import ENGINE
from pyfarm.logger import Logger
from pyfarm.datatypes.enums import State, DEFAULT_JOBTYPES, DEFAULT_GROUPS, DEFAULT_SOFTWARE

logger = Logger(__name__)

HOSTNAME_LENGTH = 255
IPV4_LENGTH = 16

# create global metadata object and bind the engine
metadata = sql.MetaData()
metadata.bind = ENGINE

hosts = sql.Table('pyfarm_hosts', metadata,
    sql.Column('id', sql.Integer, autoincrement=True, primary_key=True),
    sql.Column('hostname', sql.String(HOSTNAME_LENGTH), nullable=False),
    sql.Column('port', sql.Integer, nullable=False),
    sql.Column('master', sql.String(HOSTNAME_LENGTH), default=None),
    sql.Column('ip', sql.String(IPV4_LENGTH)),
    sql.Column('subnet', sql.String(IPV4_LENGTH)),
    sql.Column('os', sql.Integer),
    sql.Column('ram_total', sql.Integer),
    sql.Column('ram_usage', sql.Integer),
    sql.Column('swap_total', sql.Integer),
    sql.Column('swap_usage', sql.Integer),
    sql.Column('cpu_count', sql.Integer),
    sql.Column('online', sql.Boolean, nullable=False, default=True),
    sql.Column('groups', sql.PickleType, default=DEFAULT_GROUPS),
    sql.Column('software', sql.PickleType, default=DEFAULT_SOFTWARE),
    sql.Column('jobtypes', sql.PickleType, default=DEFAULT_JOBTYPES)
)

# MASTERS TABLE ATTRIBUTES
# queue - queue management and accepting new jobs
# assignment - sending jobs to hosts when enabled
masters = sql.Table('pyfarm_masters', metadata,
    sql.Column('id', sql.Integer, autoincrement=True, primary_key=True),
    sql.Column('hostname', sql.String(HOSTNAME_LENGTH), nullable=False),
    sql.Column('port', sql.Integer, nullable=False),
    sql.Column('ip', sql.String(IPV4_LENGTH)),
    sql.Column('subnet', sql.String(IPV4_LENGTH)),
    sql.Column('online', sql.Boolean, default=False),
    sql.Column('queue', sql.Boolean, default=True),
    sql.Column('assignment', sql.Boolean, default=True)
)

jobs = sql.Table('pyfarm_jobs', metadata,
    sql.Column('id', sql.Integer, autoincrement=True, primary_key=True),
    sql.Column('state', sql.Integer, default=0),
    sql.Column('priority', sql.Integer, default=prefs.get('jobsystem.priority-default')),

    # frame range declaration
    sql.Column('start_frame', sql.Integer),
    sql.Column('end_frame', sql.Integer),
    sql.Column('by_frame', sql.Integer, default=1),
    sql.Column('batch_frame', sql.Integer, default=1),

    sql.Column('time_submitted', sql.DateTime, default=datetime.datetime.now),

    # job setup
    # isolate - if True this job must run by itself (no other jobs on host)
    # cpus - number of cpus required to be free on the host
    # enviro - pickle of an environment dictionary
    sql.Column('cmd', sql.Text(convert_unicode=True), nullable=False),
    sql.Column('args', sql.PickleType, nullable=False),
    sql.Column('notes', sql.String(512, convert_unicode=True), default=""),
    sql.Column('environ', sql.PickleType),
    sql.Column('data', sql.PickleType),
    sql.Column('user', sql.String(256)),
    sql.Column('software', sql.PickleType, default=None),
    sql.Column('jobtype', sql.String(64)),
    sql.Column('ram', sql.Integer, default=None),
    sql.Column('cpus', sql.Integer, default=None),
    sql.Column('requeue_failed', sql.Boolean,
               default=prefs.get('jobtypes.defaults.requeue-failed')),
    sql.Column('requeue_max', sql.Integer,
               default=prefs.get('jobtypes.defaults.requeue-max'))
)

# FRAMES TABLE ATTRIBUTES
# host - host id currently running the frame
frames = sql.Table('pyfarm_frames', metadata,
    sql.Column('id', sql.Integer, autoincrement=True, primary_key=True),
    sql.Column('host', sql.Integer, sql.ForeignKey(hosts.c.id), default=None),
    sql.Column('jobid', sql.Integer, sql.ForeignKey(jobs.c.id)),
    sql.Column('priority', sql.Integer, default=prefs.get('jobsystem.priority-default')),
    sql.Column('host', sql.Integer, sql.ForeignKey(hosts.c.id), default=None),
    sql.Column('frame', sql.Integer),
    sql.Column('state', sql.Integer, default=State.QUEUED),
    sql.Column('attempts', sql.Integer, default=0),
    sql.Column('ram', sql.Integer, default=None),
    sql.Column('time_start', sql.DateTime, default=None),
    sql.Column('time_end', sql.DateTime, default=None),
    sql.Column('time_submitted', sql.DateTime, default=datetime.datetime.now),
    sql.Column('dependencies', sql.PickleType, default=[])
)

def init(rebuild=False):
    '''initializes the tables according the the preferences'''
    if rebuild or prefs.get('database.setup.rebuild'):
        logger.warning('dropping all tables before rebuilding')
        metadata.drop_all()

    metadata.create_all()
# end init
<commit_msg>adding jobs list so the assignment process does not require nearly as many queries<commit_after># No shebang line, this module is meant to be imported
#
# This file is part of PyFarm.
# Copyright (C) 2008-2012 Oliver Palmer
#
# PyFarm is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyFarm is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with PyFarm.  If not, see <http://www.gnu.org/licenses/>.

import datetime
import sqlalchemy as sql

from pyfarm.preferences import prefs
from pyfarm.db.session import ENGINE
from pyfarm.logger import Logger
from pyfarm.datatypes.enums import State, DEFAULT_JOBTYPES, DEFAULT_GROUPS, DEFAULT_SOFTWARE

logger = Logger(__name__)

HOSTNAME_LENGTH = 255
IPV4_LENGTH = 16

# create global metadata object and bind the engine
metadata = sql.MetaData()
metadata.bind = ENGINE

hosts = sql.Table('pyfarm_hosts', metadata,
    sql.Column('id', sql.Integer, autoincrement=True, primary_key=True),
    sql.Column('hostname', sql.String(HOSTNAME_LENGTH), nullable=False),
    sql.Column('port', sql.Integer, nullable=False),
    sql.Column('master', sql.String(HOSTNAME_LENGTH), default=None),
    sql.Column('ip', sql.String(IPV4_LENGTH)),
    sql.Column('subnet', sql.String(IPV4_LENGTH)),
    sql.Column('os', sql.Integer),
    sql.Column('ram_total', sql.Integer),
    sql.Column('ram_usage', sql.Integer),
    sql.Column('swap_total', sql.Integer),
    sql.Column('swap_usage', sql.Integer),
    sql.Column('cpu_count', sql.Integer),
    sql.Column('online', sql.Boolean, nullable=False, default=True),
    sql.Column('groups', sql.PickleType, default=DEFAULT_GROUPS),
    sql.Column('software', sql.PickleType, default=DEFAULT_SOFTWARE),
    sql.Column('jobtypes', sql.PickleType, default=DEFAULT_JOBTYPES),
    sql.Column('jobs', sql.PickleType, default=[])
)

# MASTERS TABLE ATTRIBUTES
# queue - queue management and accepting new jobs
# assignment - sending jobs to hosts when enabled
masters = sql.Table('pyfarm_masters', metadata,
    sql.Column('id', sql.Integer, autoincrement=True, primary_key=True),
    sql.Column('hostname', sql.String(HOSTNAME_LENGTH), nullable=False),
    sql.Column('port', sql.Integer, nullable=False),
    sql.Column('ip', sql.String(IPV4_LENGTH)),
    sql.Column('subnet', sql.String(IPV4_LENGTH)),
    sql.Column('online', sql.Boolean, default=False),
    sql.Column('queue', sql.Boolean, default=True),
    sql.Column('assignment', sql.Boolean, default=True)
)

jobs = sql.Table('pyfarm_jobs', metadata,
    sql.Column('id', sql.Integer, autoincrement=True, primary_key=True),
    sql.Column('state', sql.Integer, default=0),
    sql.Column('priority', sql.Integer, default=prefs.get('jobsystem.priority-default')),

    # frame range declaration
    sql.Column('start_frame', sql.Integer),
    sql.Column('end_frame', sql.Integer),
    sql.Column('by_frame', sql.Integer, default=1),
    sql.Column('batch_frame', sql.Integer, default=1),

    sql.Column('time_submitted', sql.DateTime, default=datetime.datetime.now),

    # job setup
    # isolate - if True this job must run by itself (no other jobs on host)
    # cpus - number of cpus required to be free on the host
    # enviro - pickle of an environment dictionary
    sql.Column('cmd', sql.Text(convert_unicode=True), nullable=False),
    sql.Column('args', sql.PickleType, nullable=False),
    sql.Column('notes', sql.String(512, convert_unicode=True), default=""),
    sql.Column('environ', sql.PickleType),
    sql.Column('data', sql.PickleType),
    sql.Column('user', sql.String(256)),
    sql.Column('software', sql.PickleType, default=None),
    sql.Column('jobtype', sql.String(64)),
    sql.Column('ram', sql.Integer, default=None),
    sql.Column('cpus', sql.Integer, default=None),
    sql.Column('requeue_failed', sql.Boolean,
               default=prefs.get('jobtypes.defaults.requeue-failed')),
    sql.Column('requeue_max', sql.Integer,
               default=prefs.get('jobtypes.defaults.requeue-max'))
)

# FRAMES TABLE ATTRIBUTES
# host - host id currently running the frame
frames = sql.Table('pyfarm_frames', metadata,
    sql.Column('id', sql.Integer, autoincrement=True, primary_key=True),
    sql.Column('host', sql.Integer, sql.ForeignKey(hosts.c.id), default=None),
    sql.Column('jobid', sql.Integer, sql.ForeignKey(jobs.c.id)),
    sql.Column('priority', sql.Integer, default=prefs.get('jobsystem.priority-default')),
    sql.Column('host', sql.Integer, sql.ForeignKey(hosts.c.id), default=None),
    sql.Column('frame', sql.Integer),
    sql.Column('state', sql.Integer, default=State.QUEUED),
    sql.Column('attempts', sql.Integer, default=0),
    sql.Column('ram', sql.Integer, default=None),
    sql.Column('time_start', sql.DateTime, default=None),
    sql.Column('time_end', sql.DateTime, default=None),
    sql.Column('time_submitted', sql.DateTime, default=datetime.datetime.now),
    sql.Column('dependencies', sql.PickleType, default=[])
)

def init(rebuild=False):
    '''initializes the tables according the the preferences'''
    if rebuild or prefs.get('database.setup.rebuild'):
        logger.warning('dropping all tables before rebuilding')
        metadata.drop_all()

    metadata.create_all()
# end init
<|endoftext|>


<commit_before>#!/usr/bin/env python

import time
import datetime

import rospy
from mongodb_store.message_store import MessageStoreProxy
from exploration_bid_manager.exploration_bidder import ExplorationBidder

from popp_exploration.msg import ExplorationChoice

from multi_detect_temporal_patterns.srv import BestTimeEstimateSrv
from multi_detect_temporal_patterns.srv import BestTimeEstimateSrvResponse

class BiddingManager(object):

    def __init__(
        self, start_time=(8, 0), end_time=(22, 0),
        exploration_interval=rospy.Duration(1800), minimum_bid=500
    ):
        rospy.loginfo("Initiating bid planning...")
        # hand-allocated budget (between 8am to 6pm)
        self.exploration_interval = exploration_interval
        self._start_time = datetime.time(start_time[0], start_time[1])
        self._end_time = datetime.time(end_time[0], end_time[1])
        self.bidding_plan = list()
        self.available_tokens = 0
        self.minimum_bid = minimum_bid
        tmp = datetime.datetime.fromtimestamp(rospy.Time.now().secs)
        self._bidding_date = datetime.datetime(
            tmp.year, tmp.month, tmp.day, 0, 0
        )
        self.bidder = ExplorationBidder()
        self.soma_config = rospy.get_param(
            "~soma_config", "poisson_activity"
        )
        self._db = MessageStoreProxy(collection="popp_exploration_log")
        # all services to counters
        popp_srv_name = "/multi_sensor_counting_process/best_time_estimate"
        rospy.loginfo("Connecting to %s service..." % popp_srv_name)
        self._popp_srv = rospy.ServiceProxy(popp_srv_name, BestTimeEstimateSrv)
        self._popp_srv.wait_for_service()

    def get_popp_estimate(self, start, end, size):
        result = None
        try:
            result = self._popp_srv(start, end, size, True)
        except NameError:
            result = BestTimeEstimateSrvResponse(list(), list(), list())
        return result

    def calculate_bidding_plan(self, recommended_regions=[], time_visited_regions=[]):
        current_time = datetime.datetime.fromtimestamp(
            (rospy.Time.now().secs / 86400) * 86400
        )
        available_tokens = self.bidder.available_tokens - self.bidder.currently_bid_tokens
        if current_time-self._bidding_date >= datetime.timedelta(days=1):
            rospy.loginfo(
                "Calculating bidding plan for %s..." % str(
                    current_time.date()
                )
            )
            start = datetime.datetime(
                current_time.year, current_time.month, current_time.day,
                self._start_time.hour, self._start_time.minute
            )
            end = datetime.datetime(
                current_time.year, current_time.month, current_time.day,
                self._end_time.hour, self._end_time.minute
            )
            start = rospy.Time(time.mktime(start.timetuple()))
            end = rospy.Time(time.mktime(end.timetuple()))
            self._calculate_bidding_plan(
                start, end, available_tokens, recommended_regions, time_visited_regions
            )
            self._bidding_date = current_time
        elif available_tokens:
            start = datetime.datetime.fromtimestamp(
                (rospy.Time.now().secs / 60) * 60
            )
            rospy.loginfo(
                "Calculating bidding plan with %d avaible bid from %s..." % (
                    available_tokens, str(start)
                )
            )
            # end = datetime.datetime(
            #     start.year, start.month, start.day,
            #     self._end_time.hour, self._end_time.minute
            # )
            start = rospy.Time(time.mktime(start.timetuple()))
            # end = rospy.Time(time.mktime(end.timetuple()))
            end = rospy.Time(start.secs + (2*self.exploration_interval.secs))
            self._calculate_bidding_plan(
                start, end, available_tokens, recommended_regions, time_visited_regions
            )
        else:
            rospy.loginfo("No available bid, skipping process..")

    def _calculate_bidding_plan(
        self, start, end, available_tokens, recommended_regions=[], time_visited_regions=[]
    ):
        rospy.loginfo("Available tokens: %d" % available_tokens)
        estimates = list()
        while (start + self.exploration_interval) <= end:
            estimate = self.get_popp_estimate(start, start + self.exploration_interval, 5)
            if recommended_regions != list():
                estimate = [
                    (
                        estimate.times[j], i, estimate.estimates[j]
                    ) for j, i in enumerate(estimate.region_ids) if i in recommended_regions
                ]
            if not len(estimate):
                rospy.logwarn(
                    "These regions %s are not covered by any waypoint" % str(zip(*estimate)[1])
                )
            if time_visited_regions != list():
                estimate = [i for i in estimate if (i[0], i[1]) not in time_visited_regions]
            if not len(estimate):
                rospy.loginfo("No extra exploration needed...")
            if len(estimate):
                estimates.append(estimate[0])
            start = start + self.exploration_interval
        self._merge_plan_with_tokens(estimates, available_tokens)
        if len(self.bidding_plan):
            # store options to db
            msg = ExplorationChoice()
            msg.soma_config = self.soma_config
            for i in estimates:
                if i[0] in zip(*self.bidding_plan)[0]:
                    msg.start_times.append(i[0])
                    msg.region_ids.append(i[1])
                    msg.estimates.append(i[2])
                    msg.bid.append(
                        zip(*self.bidding_plan)[2][
                            zip(*self.bidding_plan)[0].index(i[0])
                        ]
                    )
            self._db.insert(msg)
        self.available_tokens = available_tokens
        rospy.loginfo("Feasible bidding plan: %s" % str(self.bidding_plan))

    def _merge_plan_with_tokens(self, estimates, available_tokens):
        if len(estimates):
            # normalize estimate
            norm = sum(zip(*estimates)[2])
            if norm > 0.0:
                proposed_bidding_plan = [
                    (
                        i[0], i[1], i[2]
                    ) for i in estimates if int(
                        (i[2]/norm)*available_tokens
                    ) >= self.minimum_bid
                ]
                if len(proposed_bidding_plan):
                    norm = sum(zip(*proposed_bidding_plan)[2])
                else:
                    rospy.logwarn("Allocated budget does not meet the minimum bid required.")
            elif int(
                (1 / float(len(estimates))) * available_tokens
            ) >= self.minimum_bid:
                norm = float(len(estimates))
                proposed_bidding_plan = [(i[0], i[1], 1) for i in estimates]
            else:
                estimates = estimates[:len(estimates)/2]
                norm = float(len(estimates))
                proposed_bidding_plan = [(i[0], i[1], 1) for i in estimates]
            self.bidding_plan = [
                (
                    i[0], i[1], int((i[2]/norm)*available_tokens)
                ) for i in proposed_bidding_plan
            ]
        else:
            self.bidding_plan = list()
<commit_msg>reset the previous change<commit_after>#!/usr/bin/env python

import time
import datetime

import rospy
from mongodb_store.message_store import MessageStoreProxy
from exploration_bid_manager.exploration_bidder import ExplorationBidder

from popp_exploration.msg import ExplorationChoice

from multi_detect_temporal_patterns.srv import BestTimeEstimateSrv
from multi_detect_temporal_patterns.srv import BestTimeEstimateSrvResponse

class BiddingManager(object):

    def __init__(
        self, start_time=(8, 0), end_time=(22, 0),
        exploration_interval=rospy.Duration(1800), minimum_bid=500
    ):
        rospy.loginfo("Initiating bid planning...")
        # hand-allocated budget (between 8am to 6pm)
        self.exploration_interval = exploration_interval
        self._start_time = datetime.time(start_time[0], start_time[1])
        self._end_time = datetime.time(end_time[0], end_time[1])
        self.bidding_plan = list()
        self.available_tokens = 0
        self.minimum_bid = minimum_bid
        tmp = datetime.datetime.fromtimestamp(rospy.Time.now().secs)
        self._bidding_date = datetime.datetime(
            tmp.year, tmp.month, tmp.day, 0, 0
        )
        self.bidder = ExplorationBidder()
        self.soma_config = rospy.get_param(
            "~soma_config", "poisson_activity"
        )
        self._db = MessageStoreProxy(collection="popp_exploration_log")
        # all services to counters
        popp_srv_name = "/multi_sensor_counting_process/best_time_estimate"
        rospy.loginfo("Connecting to %s service..." % popp_srv_name)
        self._popp_srv = rospy.ServiceProxy(popp_srv_name, BestTimeEstimateSrv)
        self._popp_srv.wait_for_service()

    def get_popp_estimate(self, start, end, size):
        result = None
        try:
            result = self._popp_srv(start, end, size, True)
        except NameError:
            result = BestTimeEstimateSrvResponse(list(), list(), list())
        return result

    def calculate_bidding_plan(self, recommended_regions=[], time_visited_regions=[]):
        current_time = datetime.datetime.fromtimestamp(
            (rospy.Time.now().secs / 86400) * 86400
        )
        available_tokens = self.bidder.available_tokens - self.bidder.currently_bid_tokens
        if current_time-self._bidding_date >= datetime.timedelta(days=1):
            rospy.loginfo(
                "Calculating bidding plan for %s..." % str(
                    current_time.date()
                )
            )
            start = datetime.datetime(
                current_time.year, current_time.month, current_time.day,
                self._start_time.hour, self._start_time.minute
            )
            end = datetime.datetime(
                current_time.year, current_time.month, current_time.day,
                self._end_time.hour, self._end_time.minute
            )
            start = rospy.Time(time.mktime(start.timetuple()))
            end = rospy.Time(time.mktime(end.timetuple()))
            self._calculate_bidding_plan(
                start, end, available_tokens, recommended_regions, time_visited_regions
            )
            self._bidding_date = current_time
        elif available_tokens:
            start = datetime.datetime.fromtimestamp(
                (rospy.Time.now().secs / 60) * 60
            )
            rospy.loginfo(
                "Calculating bidding plan with %d avaible bid from %s..." % (
                    available_tokens, str(start)
                )
            )
            end = datetime.datetime(
                start.year, start.month, start.day,
                self._end_time.hour, self._end_time.minute
            )
            start = rospy.Time(time.mktime(start.timetuple()))
            end = rospy.Time(time.mktime(end.timetuple()))
            self._calculate_bidding_plan(
                start, end, available_tokens, recommended_regions, time_visited_regions
            )
        else:
            rospy.loginfo("No available bid, skipping process..")

    def _calculate_bidding_plan(
        self, start, end, available_tokens, recommended_regions=[], time_visited_regions=[]
    ):
        rospy.loginfo("Available tokens: %d" % available_tokens)
        estimates = list()
        while (start + self.exploration_interval) <= end:
            estimate = self.get_popp_estimate(start, start + self.exploration_interval, 5)
            if recommended_regions != list():
                estimate = [
                    (
                        estimate.times[j], i, estimate.estimates[j]
                    ) for j, i in enumerate(estimate.region_ids) if i in recommended_regions
                ]
            if not len(estimate):
                rospy.logwarn(
                    "These regions %s are not covered by any waypoint" % str(zip(*estimate)[1])
                )
            if time_visited_regions != list():
                estimate = [i for i in estimate if (i[0], i[1]) not in time_visited_regions]
            if not len(estimate):
                rospy.loginfo("No extra exploration needed...")
            if len(estimate):
                estimates.append(estimate[0])
            start = start + self.exploration_interval
        self._merge_plan_with_tokens(estimates, available_tokens)
        if len(self.bidding_plan):
            # store options to db
            msg = ExplorationChoice()
            msg.soma_config = self.soma_config
            for i in estimates:
                if i[0] in zip(*self.bidding_plan)[0]:
                    msg.start_times.append(i[0])
                    msg.region_ids.append(i[1])
                    msg.estimates.append(i[2])
                    msg.bid.append(
                        zip(*self.bidding_plan)[2][
                            zip(*self.bidding_plan)[0].index(i[0])
                        ]
                    )
            self._db.insert(msg)
        self.available_tokens = available_tokens
        rospy.loginfo("Feasible bidding plan: %s" % str(self.bidding_plan))

    def _merge_plan_with_tokens(self, estimates, available_tokens):
        if len(estimates):
            # normalize estimate
            norm = sum(zip(*estimates)[2])
            if norm > 0.0:
                proposed_bidding_plan = [
                    (
                        i[0], i[1], i[2]
                    ) for i in estimates if int(
                        (i[2]/norm)*available_tokens
                    ) >= self.minimum_bid
                ]
                if len(proposed_bidding_plan):
                    norm = sum(zip(*proposed_bidding_plan)[2])
                else:
                    rospy.logwarn("Allocated budget does not meet the minimum bid required.")
            elif int(
                (1 / float(len(estimates))) * available_tokens
            ) >= self.minimum_bid:
                norm = float(len(estimates))
                proposed_bidding_plan = [(i[0], i[1], 1) for i in estimates]
            else:
                estimates = estimates[:len(estimates)/2]
                norm = float(len(estimates))
                proposed_bidding_plan = [(i[0], i[1], 1) for i in estimates]
            self.bidding_plan = [
                (
                    i[0], i[1], int((i[2]/norm)*available_tokens)
                ) for i in proposed_bidding_plan
            ]
        else:
            self.bidding_plan = list()
<|endoftext|>