<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# This file is part of PyBOSSA.
#
# PyBOSSA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBOSSA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBOSSA.  If not, see <http://www.gnu.org/licenses/>.
"""
Helper functions for the pbs command line client.

This module exports the following methods:
    * find_app_by_short_name: return the project by short_name.
    * check_api_errors: check for API errors returned by a PyBossa server.
    * format_error: format error message.
    * format_json_task: format a CSV row into JSON.
"""
import csv
import json
import time
import click
import logging
import StringIO
import polib
from requests import exceptions
from pbsexceptions import ProjectNotFound, TaskNotFound

__all__ = ['find_app_by_short_name', 'check_api_error',
           'format_error', 'format_json_task', '_create_project',
           '_update_project', '_add_tasks', 'create_task_info',
           '_delete_tasks', 'enable_auto_throttling',
           '_update_tasks_redundancy']

def _create_project(config):
    """Create a project in a PyBossa server."""
    try:
        response = config.pbclient.create_app(config.project['name'],
                                              config.project['short_name'],
                                              config.project['description'])
        check_api_error(response)
        return ("Project: %s created!" % config.project['short_name'])
    except exceptions.ConnectionError:
        return("Connection Error! The server %s is not responding" % config.server)
    except ProjectNotFound, TaskNotFound:
        raise

def _update_project(config, task_presenter, long_description, tutorial):
    """Update a project."""
    try:
        # Get project
        project = find_app_by_short_name(config.project['short_name'],
                                         config.pbclient)
        # Update attributes
        project.name = config.project['name']
        project.short_name = config.project['short_name']
        project.description = config.project['description']
        project.long_description = long_description.read()
        # Update task presenter
        project.info['task_presenter'] = task_presenter.read()
        # Update tutorial
        project.info['tutorial'] = tutorial.read()
        response = config.pbclient.update_app(project)
        check_api_error(response)
        return ("Project %s updated!" % config.project['short_name'])
    except exceptions.ConnectionError:
        return ("Connection Error! The server %s is not responding" % config.server)
    except ProjectNotFound, TaskNotFound:
        raise

def _add_tasks(config, tasks_file, tasks_type, priority, redundancy):
    """Add tasks to a project."""
    try:
        project = find_app_by_short_name(config.project['short_name'],
                                         config.pbclient)
        tasks = tasks_file.read()
        # Data list to process
        data = []
        # JSON type
        if tasks_type == 'json':
            data = json.loads(tasks)
        # CSV type
        elif tasks_type == 'csv':
            csv_data = StringIO.StringIO(tasks)
            reader = csv.DictReader(csv_data, delimiter=',')
            n_tasks = 0
            for line in reader:
                data.append(line)
        # PO type
        elif tasks_type == 'po':
            po = polib.pofile(tasks)
            for entry in po.untranslated_entries():
                data.append(entry.__dict__)
        # PROPERTIES type (used in Java and Firefox extensions)
        elif tasks_type == 'properties':
            lines = tasks.split('\n')
            for l in lines:
                if l:
                    var_id, string = l.split('=')
                    tmp = dict(var_id=var_id, string=string)
                    data.append(tmp)
        else:
            return ("Unknown format for the tasks file. Use json, csv or po.")
        # Check if for the data we have to auto-throttle task creation
        sleep, msg = enable_auto_throttling(data)
        # If true, warn user
        if sleep: # pragma: no cover
            click.secho(msg, fg='yellow')
        # Show progress bar
        with click.progressbar(data, label="Adding Tasks") as pgbar:
            for d in pgbar:
                task_info = create_task_info(d)
                response = config.pbclient.create_task(app_id=project.id,
                                                       info=task_info,
                                                       n_answers=redundancy,
                                                       priority_0=priority)
                check_api_error(response)
                # If auto-throttling enabled, sleep for 3 seconds
                if sleep: # pragma: no cover
                    time.sleep(3)
            return ("%s tasks added to project: %s" % (len(data),
                                                  config.project['short_name']))
    except exceptions.ConnectionError:
        return ("Connection Error! The server %s is not responding" % config.server)
    except ProjectNotFound, TaskNotFound:
        raise

def _delete_tasks(config, task_id, limit=100, offset=0):
    """Delete tasks from a project."""
    try:
        project = find_app_by_short_name(config.project['short_name'],
                                         config.pbclient)
        if task_id:
            response = config.pbclient.delete_task(task_id)
            check_api_error(response)
            return "Task.id = %s and its associated task_runs have been deleted" % task_id
        else:
            limit = limit
            offset = offset
            tasks = config.pbclient.get_tasks(project.id, limit, offset)
            while len(tasks) > 0:
                for t in tasks:
                    response = config.pbclient.delete_task(t.id)
                    check_api_error(response)
                offset += limit
                tasks = config.pbclient.get_tasks(project.id, limit, offset)
            return "All tasks and task_runs have been deleted"
    except exceptions.ConnectionError:
        return ("Connection Error! The server %s is not responding" % config.server)
    except ProjectNotFound, TaskNotFound:
        raise

def _update_tasks_redundancy(config, task_id, redundancy, limit=300, offset=0):
    """Update tasks redundancy from a project."""
    try:
        project = find_app_by_short_name(config.project['short_name'],
                                         config.pbclient)
        if task_id:
            response = config.pbclient.find_tasks(project.id, id=task_id)
            check_api_error(response)
            task = response[0]
            task.n_answers = redundancy
            response = config.pbclient.update_task(task)
            check_api_error(response)
            msg = "Task.id = %s redundancy has been updated to %s" % (task_id,
                                                                      redundancy)
            return msg
        else:
            limit = limit
            offset = offset
            tasks = config.pbclient.get_tasks(project.id, limit, offset)
            # Check if for the data we have to auto-throttle task update
            sleep, msg = enable_auto_throttling(tasks)
            # If true, warn user
            if sleep: # pragma: no cover
                click.secho(msg, fg='yellow')
            with click.progressbar(tasks, label="Updating Tasks") as pgbar:
                while len(tasks) > 0:
                    for t in pgbar:
                        t.n_answers = redundancy
                        response = config.pbclient.update_task(t)
                        check_api_error(response)
                        # If auto-throttling enabled, sleep for 3 seconds
                        if sleep: # pragma: no cover
                            time.sleep(3)
                    offset += limit
                    tasks = config.pbclient.get_tasks(project.id, limit, offset)
                return "All tasks redundancy have been updated"
    except exceptions.ConnectionError:
        return ("Connection Error! The server %s is not responding" % config.server)
    except ProjectNotFound, TaskNotFound:
        raise

def find_app_by_short_name(short_name, pbclient):
    """Return project by short_name."""
    try:
        response = pbclient.find_app(short_name=short_name)
        check_api_error(response)
        return response[0]
    except exceptions.ConnectionError:
        raise
    except ProjectNotFound:
        raise

def check_api_error(api_response):
    """Check if returned API response contains an error."""
    if type(api_response) == dict and (api_response.get('status') == 'failed'):
        if 'app' in api_response.get('target'):
            raise ProjectNotFound(message='PyBossa Project not found',
                                  error=api_response)
        if 'task' in api_response.get('target'):
            raise TaskNotFound(message='PyBossa Task not found',
                               error=api_response)
        else:
            raise exceptions.HTTPError

def format_error(module, error):
    """Format the error for the given module."""
    logging.error(module)
    # Beautify JSON error
    print error.message
    print json.dumps(error.error, sort_keys=True, indent=4, separators=(',', ': '))
    exit(1)

def create_task_info(task):
    """Create task_info field."""
    task_info = None
    if task.get('info'):
        task_info = task['info']
    else:
        task_info = task
    return task_info

def enable_auto_throttling(data, limit=299):
    """Return True if more than 300 tasks have to be created."""
    msg = 'Warning: %s tasks to create.' \
          ' Auto-throttling enabled!' % len(data)
    if len(data) > limit:
        return (True, msg)
    else:
        return False, None

def format_json_task(task_info):
    """Format task_info into JSON if applicable."""
    try:
        return json.loads(task_info)
    except:
        return task_info
<commit_msg>Fix error with exceptions.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# This file is part of PyBOSSA.
#
# PyBOSSA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBOSSA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBOSSA.  If not, see <http://www.gnu.org/licenses/>.
"""
Helper functions for the pbs command line client.

This module exports the following methods:
    * find_app_by_short_name: return the project by short_name.
    * check_api_errors: check for API errors returned by a PyBossa server.
    * format_error: format error message.
    * format_json_task: format a CSV row into JSON.
"""
import csv
import json
import time
import click
import logging
import StringIO
import polib
from requests import exceptions
from pbsexceptions import ProjectNotFound, TaskNotFound

__all__ = ['find_app_by_short_name', 'check_api_error',
           'format_error', 'format_json_task', '_create_project',
           '_update_project', '_add_tasks', 'create_task_info',
           '_delete_tasks', 'enable_auto_throttling',
           '_update_tasks_redundancy']

def _create_project(config):
    """Create a project in a PyBossa server."""
    try:
        response = config.pbclient.create_app(config.project['name'],
                                              config.project['short_name'],
                                              config.project['description'])
        check_api_error(response)
        return ("Project: %s created!" % config.project['short_name'])
    except exceptions.ConnectionError:
        return("Connection Error! The server %s is not responding" % config.server)
    except (ProjectNotFound, TaskNotFound):
        raise

def _update_project(config, task_presenter, long_description, tutorial):
    """Update a project."""
    try:
        # Get project
        project = find_app_by_short_name(config.project['short_name'],
                                         config.pbclient)
        # Update attributes
        project.name = config.project['name']
        project.short_name = config.project['short_name']
        project.description = config.project['description']
        project.long_description = long_description.read()
        # Update task presenter
        project.info['task_presenter'] = task_presenter.read()
        # Update tutorial
        project.info['tutorial'] = tutorial.read()
        response = config.pbclient.update_app(project)
        check_api_error(response)
        return ("Project %s updated!" % config.project['short_name'])
    except exceptions.ConnectionError:
        return ("Connection Error! The server %s is not responding" % config.server)
    except (ProjectNotFound, TaskNotFound):
        raise

def _add_tasks(config, tasks_file, tasks_type, priority, redundancy):
    """Add tasks to a project."""
    try:
        project = find_app_by_short_name(config.project['short_name'],
                                         config.pbclient)
        tasks = tasks_file.read()
        # Data list to process
        data = []
        # JSON type
        if tasks_type == 'json':
            data = json.loads(tasks)
        # CSV type
        elif tasks_type == 'csv':
            csv_data = StringIO.StringIO(tasks)
            reader = csv.DictReader(csv_data, delimiter=',')
            n_tasks = 0
            for line in reader:
                data.append(line)
        # PO type
        elif tasks_type == 'po':
            po = polib.pofile(tasks)
            for entry in po.untranslated_entries():
                data.append(entry.__dict__)
        # PROPERTIES type (used in Java and Firefox extensions)
        elif tasks_type == 'properties':
            lines = tasks.split('\n')
            for l in lines:
                if l:
                    var_id, string = l.split('=')
                    tmp = dict(var_id=var_id, string=string)
                    data.append(tmp)
        else:
            return ("Unknown format for the tasks file. Use json, csv or po.")
        # Check if for the data we have to auto-throttle task creation
        sleep, msg = enable_auto_throttling(data)
        # If true, warn user
        if sleep: # pragma: no cover
            click.secho(msg, fg='yellow')
        # Show progress bar
        with click.progressbar(data, label="Adding Tasks") as pgbar:
            for d in pgbar:
                task_info = create_task_info(d)
                response = config.pbclient.create_task(app_id=project.id,
                                                       info=task_info,
                                                       n_answers=redundancy,
                                                       priority_0=priority)
                check_api_error(response)
                # If auto-throttling enabled, sleep for 3 seconds
                if sleep: # pragma: no cover
                    time.sleep(3)
            return ("%s tasks added to project: %s" % (len(data),
                                                  config.project['short_name']))
    except exceptions.ConnectionError:
        return ("Connection Error! The server %s is not responding" % config.server)
    except (ProjectNotFound, TaskNotFound):
        raise

def _delete_tasks(config, task_id, limit=100, offset=0):
    """Delete tasks from a project."""
    try:
        project = find_app_by_short_name(config.project['short_name'],
                                         config.pbclient)
        if task_id:
            response = config.pbclient.delete_task(task_id)
            check_api_error(response)
            return "Task.id = %s and its associated task_runs have been deleted" % task_id
        else:
            limit = limit
            offset = offset
            tasks = config.pbclient.get_tasks(project.id, limit, offset)
            while len(tasks) > 0:
                for t in tasks:
                    response = config.pbclient.delete_task(t.id)
                    check_api_error(response)
                offset += limit
                tasks = config.pbclient.get_tasks(project.id, limit, offset)
            return "All tasks and task_runs have been deleted"
    except exceptions.ConnectionError:
        return ("Connection Error! The server %s is not responding" % config.server)
    except (ProjectNotFound, TaskNotFound):
        raise

def _update_tasks_redundancy(config, task_id, redundancy, limit=300, offset=0):
    """Update tasks redundancy from a project."""
    try:
        project = find_app_by_short_name(config.project['short_name'],
                                         config.pbclient)
        if task_id:
            response = config.pbclient.find_tasks(project.id, id=task_id)
            check_api_error(response)
            task = response[0]
            task.n_answers = redundancy
            response = config.pbclient.update_task(task)
            check_api_error(response)
            msg = "Task.id = %s redundancy has been updated to %s" % (task_id,
                                                                      redundancy)
            return msg
        else:
            limit = limit
            offset = offset
            tasks = config.pbclient.get_tasks(project.id, limit, offset)
            # Check if for the data we have to auto-throttle task update
            sleep, msg = enable_auto_throttling(tasks)
            # If true, warn user
            if sleep: # pragma: no cover
                click.secho(msg, fg='yellow')
            with click.progressbar(tasks, label="Updating Tasks") as pgbar:
                while len(tasks) > 0:
                    for t in pgbar:
                        t.n_answers = redundancy
                        response = config.pbclient.update_task(t)
                        check_api_error(response)
                        # If auto-throttling enabled, sleep for 3 seconds
                        if sleep: # pragma: no cover
                            time.sleep(3)
                    offset += limit
                    tasks = config.pbclient.get_tasks(project.id, limit, offset)
                return "All tasks redundancy have been updated"
    except exceptions.ConnectionError:
        return ("Connection Error! The server %s is not responding" % config.server)
    except (ProjectNotFound, TaskNotFound):
        raise

def find_app_by_short_name(short_name, pbclient):
    """Return project by short_name."""
    try:
        response = pbclient.find_app(short_name=short_name)
        check_api_error(response)
        return response[0]
    except exceptions.ConnectionError:
        raise
    except ProjectNotFound:
        raise

def check_api_error(api_response):
    """Check if returned API response contains an error."""
    if type(api_response) == dict and (api_response.get('status') == 'failed'):
        if 'app' in api_response.get('target'):
            raise ProjectNotFound(message='PyBossa Project not found',
                                  error=api_response)
        if 'task' in api_response.get('target'):
            raise TaskNotFound(message='PyBossa Task not found',
                               error=api_response)
        else:
            raise exceptions.HTTPError

def format_error(module, error):
    """Format the error for the given module."""
    logging.error(module)
    # Beautify JSON error
    print error.message
    print json.dumps(error.error, sort_keys=True, indent=4, separators=(',', ': '))
    exit(1)

def create_task_info(task):
    """Create task_info field."""
    task_info = None
    if task.get('info'):
        task_info = task['info']
    else:
        task_info = task
    return task_info

def enable_auto_throttling(data, limit=299):
    """Return True if more than 300 tasks have to be created."""
    msg = 'Warning: %s tasks to create.' \
          ' Auto-throttling enabled!' % len(data)
    if len(data) > limit:
        return (True, msg)
    else:
        return False, None

def format_json_task(task_info):
    """Format task_info into JSON if applicable."""
    try:
        return json.loads(task_info)
    except:
        return task_info
<|endoftext|>


<commit_before>#! /usr/bin/env python3.4

import os
import sqlite3
from datetime import datetime

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")

from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()

from data.models import File as DB_FILE, Path as DB_PATH
from django.db.utils import OperationalError

class InterfaceHelper(object):
    """Handles the messaging to the user."""

    def __init__(self, progress_bar, progress_var, update_idletasks, status_config):
        progress_bar.configure(maximum=100)
        self.progress_bar = progress_bar
        self.progress_var = progress_var
        self.update_idletasks = update_idletasks
        self.status_config = status_config

    def message_user(self, through='status', msg='Ready', weight=0, value=0):
        """Show a message to the user."""
        if through == 'status':
            self._use_status(msg, weight)
        if through == 'progress_bar':
            self._use_progress_bar(weight, value)
        if through == 'both':
            self._use_status(msg, weight)
            self._use_progress_bar(weight, value)

    def _use_status(self, msg, weight):
        if weight == 0:
            self.status_config(foreground='black', text=' {}'.format(str(msg)))
        if weight == 1:
            self.status_config(foreground='blue', text=' {}'.format(str(msg)))
        if weight == 2:
            self.status_config(foreground='red',
                               text=' {}'.format(str(msg)))

    def _use_progress_bar(self, weight, value):
        color = 'blue'
        if weight == 1:
            color = 'green'
        self.progress_var.set(value)
        self.progress_bar.configure(
            style="{}.Horizontal.TProgressbar".format(color))
        self.update_idletasks()

class DatabaseHelper(object):

    def __init__(self, db_name):
        self.DB_NAME = db_name
        self.db_file_objects = DB_FILE.objects
        self.db_path_objects = DB_PATH.objects
        self.db_ready = False

    def initialise_db(self):
        """Initialise database, set self.db_ready to True."""
        if not self.db_ready:
            connection = sqlite3.connect(self.DB_NAME)
            cursor = connection.cursor()
            # Create tables
            # Do not alter the queries, may not work with subsequent database
            # operations
            query1 = """CREATE TABLE IF NOT EXISTS "data_file" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "filename" text NOT NULL, "filepath_hash" text NOT NULL, "last_modified" datetime NOT NULL, "added_at" datetime NOT NULL);"""
            query2 = """CREATE TABLE IF NOT EXISTS "data_path" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "source" text NOT NULL, "destination" text NOT NULL, "accepted" bool NOT NULL, "added_at" datetime NOT NULL, "filename_id" integer NOT NULL REFERENCES "data_file" ("id"));"""
            query3 = """CREATE INDEX IF NOT EXISTS "data_path_filename_id_1d40e5f2" ON "data_path" ("filename_id");"""
            cursor.execute(query1)
            cursor.execute(query2)
            cursor.execute(query3)
            connection.commit()
            connection.close()
            self.db_ready = True

        return self.db_ready

    def get_start_value(self):
        try:
            start_value = self.db_file_objects.last().id
        except AttributeError:
            start_value = 0
        finally:
            return start_value

    def get_report(self, start_value):
        paths = self.db_path_objects.filter(
            id__gte=start_value).order_by('-pk')

        report = []
        for path in paths:
            row_tup = (path.filename.filename, path.source,
                       path.destination, path.added_at)
            report.append(row_tup)
        return report

    def update(self, database_dict):
        now = datetime.now()
        item_count = len(database_dict)

        run_list = []
        for filename in database_dict.keys():
            filename_dict = database_dict[filename]
            this_file = DB_FILE(added_at=now, **filename_dict['file'])
            run_list.append(this_file)
        self.db_file_objects.bulk_create(run_list)

        file_objects = self.db_file_objects.filter(added_at=now)

        for file_ in file_objects:
            this_path = self.db_path_objects.create(
                filename=file_, added_at=now, **database_dict[file_.filename]['path'])

    def alter_path(self, alter_value, finders):
        updated = self.db_path_objects.filter(**finders).update(**alter_value)

    def get_history(self, count):
        files = self.db_file_objects.all().order_by(
            '-pk').select_related().filter(filename_path__accepted=True)[:count]

        try:
            length = len(files)
        except OperationalError:
            files = None
        finally:
            return files
<commit_msg>Fix: empty queryset<commit_after>#! /usr/bin/env python3.4

import os
import sqlite3
from datetime import datetime

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")

from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()

from data.models import File as DB_FILE, Path as DB_PATH
from django.db.utils import OperationalError

class InterfaceHelper(object):
    """Handles the messaging to the user."""

    def __init__(self, progress_bar, progress_var, update_idletasks, status_config):
        progress_bar.configure(maximum=100)
        self.progress_bar = progress_bar
        self.progress_var = progress_var
        self.update_idletasks = update_idletasks
        self.status_config = status_config

    def message_user(self, through='status', msg='Ready', weight=0, value=0):
        """Show a message to the user."""
        if through == 'status':
            self._use_status(msg, weight)
        if through == 'progress_bar':
            self._use_progress_bar(weight, value)
        if through == 'both':
            self._use_status(msg, weight)
            self._use_progress_bar(weight, value)

    def _use_status(self, msg, weight):
        if weight == 0:
            self.status_config(foreground='black', text=' {}'.format(str(msg)))
        if weight == 1:
            self.status_config(foreground='blue', text=' {}'.format(str(msg)))
        if weight == 2:
            self.status_config(foreground='red',
                               text=' {}'.format(str(msg)))

    def _use_progress_bar(self, weight, value):
        color = 'blue'
        if weight == 1:
            color = 'green'
        self.progress_var.set(value)
        self.progress_bar.configure(
            style="{}.Horizontal.TProgressbar".format(color))
        self.update_idletasks()

class DatabaseHelper(object):

    def __init__(self, db_name):
        self.DB_NAME = db_name
        self.db_file_objects = DB_FILE.objects
        self.db_path_objects = DB_PATH.objects
        self.db_ready = False

    def initialise_db(self):
        """Initialise database, set self.db_ready to True."""
        if not self.db_ready:
            connection = sqlite3.connect(self.DB_NAME)
            cursor = connection.cursor()
            # Create tables
            # Do not alter the queries, may not work with subsequent database
            # operations
            query1 = """CREATE TABLE IF NOT EXISTS "data_file" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "filename" text NOT NULL, "filepath_hash" text NOT NULL, "last_modified" datetime NOT NULL, "added_at" datetime NOT NULL);"""
            query2 = """CREATE TABLE IF NOT EXISTS "data_path" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "source" text NOT NULL, "destination" text NOT NULL, "accepted" bool NOT NULL, "added_at" datetime NOT NULL, "filename_id" integer NOT NULL REFERENCES "data_file" ("id"));"""
            query3 = """CREATE INDEX IF NOT EXISTS "data_path_filename_id_1d40e5f2" ON "data_path" ("filename_id");"""
            cursor.execute(query1)
            cursor.execute(query2)
            cursor.execute(query3)
            connection.commit()
            connection.close()
            self.db_ready = True

        return self.db_ready

    def get_start_value(self):
        try:
            start_value = self.db_file_objects.last().id
        except AttributeError:
            start_value = 0
        finally:
            return start_value

    def get_report(self, start_value):
        paths = self.db_path_objects.filter(
            id__gte=start_value).order_by('-pk')

        report = []
        for path in paths:
            row_tup = (path.filename.filename, path.source,
                       path.destination, path.added_at)
            report.append(row_tup)
        return report

    def update(self, database_dict):
        now = datetime.now()
        item_count = len(database_dict)

        run_list = []
        for filename in database_dict.keys():
            filename_dict = database_dict[filename]
            this_file = DB_FILE(added_at=now, **filename_dict['file'])
            run_list.append(this_file)
        self.db_file_objects.bulk_create(run_list)

        file_objects = self.db_file_objects.filter(added_at=now)

        for file_ in file_objects:
            this_path = self.db_path_objects.create(
                filename=file_, added_at=now, **database_dict[file_.filename]['path'])

    def alter_path(self, alter_value, finders):
        updated = self.db_path_objects.filter(**finders).update(**alter_value)

    def get_history(self, count):
        files = self.db_file_objects.all().order_by(
            '-pk').select_related().filter(filename_path__accepted=True)[:count]

        try:
            length = len(files)
        except OperationalError:
            pass
        finally:
            return files
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Load and display Array data file, or raw SD-card-dumped disparate dumps.

...the latter is "raw" in the sense that it is not in the standard
file container for Array data as described in the technical manual.

Use the command-line flag -l to get a figure with an informative title
(at the possible cost of screen real estate).

Internal notes: There are some internal constants that should either
be dynamically determined from the given data or set by the User.

Scott Livingston  <slivingston@caltech.edu>
Jun, Sep-Oct 2010.
"""

import sys
import math
from os import getcwd

import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt

import batstack

Ts = 3.75e-6

# Before everything, look for and extract -l and -s arguments
try:
    ind = sys.argv.index('-l')
    verbose_title = True
    sys.argv.pop(ind) # Pull it out
except ValueError:
    verbose_title = False
try:
    ind = sys.argv.index('-s')
    mk_specgram = True
    sys.argv.pop(ind) # Pull it out
except ValueError:
    mk_specgram = False

# Now usual command-line argument processing
if '-h' in sys.argv:
#or (len(sys.argv) != 2 and len(sys.argv) < 4) or not (len(sys.argv)%2 == 0 or (len(sys.argv) >= 7 and sys.argv[-3] == '-t')):
    print 'Usage: %s $Array-data-file [-l] [-s]\nUsage: %s $basename ($addr $trial)^? [-t $t_start $t_stop] [-l] [-s]' % (sys.argv[0], sys.argv[0])
    exit(1)

# Handle case of reading an Array data file (rather than disparate SD-card-dumped shit).
if len(sys.argv) == 2 or (len(sys.argv) == 5 and '.bin' in sys.argv[1]):
    using_Arr_datafile = True # To help with verbose_title
    bsaf = batstack.BSArrayFile(sys.argv[1])
    bsaf.printparams()
    if len(bsaf.data) < 1:
        print 'Error reading Array data file (or its empty).'
        exit(-1)
    x = bsaf.export_chandata()

    try:
        ind = sys.argv.index('-t')
        if len(sys.argv)-ind > 2:
            t_win = [float(sys.argv[ind+1]), float(sys.argv[ind+2])]
        else:
            print 'Invalid time spec.'
            exit(-1)
    except ValueError:
        t_win = []

    nz_chans = bsaf.getnz() # Distinguish nonzero channels.
else:
    using_Arr_datafile = False # To help with verbose_title
    bname = sys.argv[1]
    bs_id = []
    trial_num = []
    len_argv = len(sys.argv)
    if len_argv%2 == 1: # Extract desired time range?
        t_win = [float(sys.argv[-2]), float(sys.argv[-1])]
        len_argv -= 3
    else:
        t_win = [] # Empty indicates use all available
    for k in range(2, len_argv, 2): # Read Stack IDs and trial numbers
        bs_id.append( int(sys.argv[k]) )
        trial_num.append( int(sys.argv[k+1]) )

    x = batstack.raw_arr_read(bname, bs_id, trial_num)
    if len(x) == 0:
        print 'Error occurred while loading trial data.'
        exit(-1)
    nz_chans = range(len(x)+1)[1:]

t = [k*Ts for k in range(-len(x[0])+1,1)]

if len(t_win) == 2:
    win_ind = batstack.find_intv(t, t_win[0], t_win[1])
    if len(win_ind) == 2:
        t = t[win_ind[0]:(win_ind[1]+1)]
        for k in range(len(x)):
            x[k] = x[k][win_ind[0]:(win_ind[1]+1)]

num_cols = int(math.ceil(len(x)/4.))
ax1 = plt.subplot(4,num_cols,1)
for ind in range(len(x)):
    ax = plt.subplot(4,num_cols,1+ind, sharex=ax1)
    for label in ax.xaxis.get_ticklabels():
        label.set_fontsize(8)
    for label in ax.yaxis.get_ticklabels():
        label.set_fontsize(8)
    if mk_specgram:
        if ind+1 in nz_chans:
            plt.specgram(x[ind]-np.mean(x[ind]), xextent=(t[0], t[-1]),
                         NFFT=128,
                         noverlap=120,
                         Fs=1./Ts)
    else:
        plt.plot(t, x[ind])
    plt.xlim([t[0], t[-1]])
    if mk_specgram and ind+1 not in nz_chans:
        pass
    else:
        plt.grid()
    if verbose_title:
        if using_Arr_datafile:
            plt.title('ch '+str(ind+1), fontsize=10)
        else:
            plt.title(str(bs_id[ind/4])+':'+str((ind%4)+1)+' (trial '+str(trial_num[ind/4])+')', fontsize=10)
if verbose_title:
    if mk_specgram:
        plt.suptitle(getcwd() + ' \n' + ' '.join(sys.argv[1:]) + '  (Subplot color axes are not equally scaled!)')
    else:
        plt.suptitle(getcwd() + ' \n' + ' '.join(sys.argv[1:]))
plt.show()
<commit_msg>Improve command-line argument processing in loadtrial.py<commit_after>#!/usr/bin/env python
"""
Load and display Array data file, or raw SD-card-dumped disparate dumps.

...the latter is "raw" in the sense that it is not in the standard
file container for Array data as described in the technical manual.

Use the command-line flag -l to get a figure with an informative title
(at the possible cost of screen real estate).

Internal notes: There are some internal constants that should either
be dynamically determined from the given data or set by the User.

Scott Livingston  <slivingston@caltech.edu>
Jun, Sep-Oct 2010.
"""

import sys
import math
from os import getcwd

import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt

import batstack

Ts = 3.75e-6

# Before everything, look for and extract -l and -s arguments
try:
    ind = sys.argv.index('-l')
    verbose_title = True
    sys.argv.pop(ind) # Pull it out
except ValueError:
    verbose_title = False
try:
    ind = sys.argv.index('-s')
    mk_specgram = True
    sys.argv.pop(ind) # Pull it out
except ValueError:
    mk_specgram = False

# Now usual command-line argument processing
if '-h' in sys.argv or len(sys.argv) == 1:
#or (len(sys.argv) != 2 and len(sys.argv) < 4) or not (len(sys.argv)%2 == 0 or (len(sys.argv) >= 7 and sys.argv[-3] == '-t')):
    ind = sys.argv[0].rfind('/')
    if ind != -1:
        progname = sys.argv[0][ind+1:]
    else:
        progname = sys.argv[0]
    print 'Usage: %s $Array-data-file [-l] [-s]\nUsage: %s $basename ($addr $trial)^? [-t $t_start $t_stop] [-l] [-s]' % (progname, progname)
    exit(1)

# Handle case of reading an Array data file (rather than disparate SD-card-dumped shit).
if len(sys.argv) == 2 or (len(sys.argv) == 5 and '.bin' in sys.argv[1]):
    using_Arr_datafile = True # To help with verbose_title
    bsaf = batstack.BSArrayFile(sys.argv[1])
    bsaf.printparams()
    if len(bsaf.data) < 1:
        print 'Error reading Array data file (or its empty).'
        exit(-1)
    x = bsaf.export_chandata()

    try:
        ind = sys.argv.index('-t')
        if len(sys.argv)-ind > 2:
            t_win = [float(sys.argv[ind+1]), float(sys.argv[ind+2])]
        else:
            print 'Invalid time spec.'
            exit(-1)
    except ValueError:
        t_win = []

    nz_chans = bsaf.getnz() # Distinguish nonzero channels.
else:
    using_Arr_datafile = False # To help with verbose_title
    bname = sys.argv[1]
    bs_id = []
    trial_num = []
    len_argv = len(sys.argv)
    if len_argv%2 == 1: # Extract desired time range?
        t_win = [float(sys.argv[-2]), float(sys.argv[-1])]
        len_argv -= 3
    else:
        t_win = [] # Empty indicates use all available
    for k in range(2, len_argv, 2): # Read Stack IDs and trial numbers
        bs_id.append( int(sys.argv[k]) )
        trial_num.append( int(sys.argv[k+1]) )

    x = batstack.raw_arr_read(bname, bs_id, trial_num)
    if len(x) == 0:
        print 'Error occurred while loading trial data.'
        exit(-1)
    nz_chans = range(len(x)+1)[1:]

t = [k*Ts for k in range(-len(x[0])+1,1)]

if len(t_win) == 2:
    win_ind = batstack.find_intv(t, t_win[0], t_win[1])
    if len(win_ind) == 2:
        t = t[win_ind[0]:(win_ind[1]+1)]
        for k in range(len(x)):
            x[k] = x[k][win_ind[0]:(win_ind[1]+1)]

num_cols = int(math.ceil(len(x)/4.))
ax1 = plt.subplot(4,num_cols,1)
for ind in range(len(x)):
    ax = plt.subplot(4,num_cols,1+ind, sharex=ax1)
    for label in ax.xaxis.get_ticklabels():
        label.set_fontsize(8)
    for label in ax.yaxis.get_ticklabels():
        label.set_fontsize(8)
    if mk_specgram:
        if ind+1 in nz_chans:
            plt.specgram(x[ind]-np.mean(x[ind]), xextent=(t[0], t[-1]),
                         NFFT=128,
                         noverlap=120,
                         Fs=1./Ts)
    else:
        plt.plot(t, x[ind])
    plt.xlim([t[0], t[-1]])
    if mk_specgram and ind+1 not in nz_chans:
        pass
    else:
        plt.grid()
    if verbose_title:
        if using_Arr_datafile:
            plt.title('ch '+str(ind+1), fontsize=10)
        else:
            plt.title(str(bs_id[ind/4])+':'+str((ind%4)+1)+' (trial '+str(trial_num[ind/4])+')', fontsize=10)
if verbose_title:
    if mk_specgram:
        plt.suptitle(getcwd() + ' \n' + ' '.join(sys.argv[1:]) + '  (Subplot color axes are not equally scaled!)')
    else:
        plt.suptitle(getcwd() + ' \n' + ' '.join(sys.argv[1:]))
plt.show()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Helper functions by Ian Huston
    $Id: helpers.py,v 1.19 2010/01/18 16:50:34 ith Exp $

    Provides helper functions for use elsewhere"""

from __future__ import division # Get rid of integer division problems, i.e. 1/2=0
import numpy as np
import re
from scipy import integrate
import os
import os.path
import logging

def nanfillstart(a, l):
    """Return an array of length l by appending array a to end of block of NaNs along axis 0."""
    if len(a) >= l:
        return a #Array already as long or longer than required
    else:
        bshape = np.array(a.shape)
        bshape[0] = l - bshape[0]
        b = np.ones(bshape)*np.NaN
        c = np.concatenate((b,a))
        return c

def eto10(number):
    """Convert scientific notation e.g. 1e-5 to 1x10^{-5} for use in LaTeX, converting to string."""
    s = re.sub(r'e(\S\d+)', r'\\times 10^{\1}', str(number))
    return s

def klegend(ks,mpc=False):
    """Return list of string representations of k modes for legend."""
    klist = []
    for k in ks:
        if mpc:
            str = r"$k=" + eto10(k) + r"M_{\mathrm{PL}} = " + eto10(mpl2invmpc(k)) + r" M\mathrm{pc}$"
        else:
            str = r"$k=" + eto10(k) + r"M_{\mathrm{PL}}$"
        klist.append(str)
    return klist

def invmpc2mpl(x=1):
    """Convert from Mpc^-1 to Mpl (reduced Planck Mass)"""
    return 2.625e-57*x

def mpl2invmpc(x=1):
    """Convert from Mpl (reduced Planck Mass) to Mpc^-1"""
    return 3.8095e+56*x

def ispower2(n):
    """Returns the log base 2 of n if n is a power of 2, zero otherwise.

    Note the potential ambiguity if n==1: 2**0==1, interpret accordingly."""

    bin_n = np.binary_repr(n)[1:]
    if '1' in bin_n:
        return 0
    else:
        return len(bin_n)

def removedups(l):
    """Return an array with duplicates removed but order retained.

    An array is returned no matter what the input type. The first of each duplicate is retained.

    Parameters
    ----------
    l: array_like
       Array (or list etc.) of values with duplicates that are to be removed.

    Returns
    -------
    retlist: ndarray
             Array of values with duplicates removed but order intact.
    """
    retlist = np.array([])
    for x in l:
        if x not in retlist:
            retlist = np.append(retlist, x)
    return retlist

def getintfunc(x):
    """Return the correct function to integrate with.

    Checks the given set of values and returns either scipy.integrate.romb
    or scipy.integrate.simps. This depends on whether the number of values is a
    power of 2 + 1 as required by romb.

    Parameters
    ----------
    x: array_like
       Array of x values to check

    Returns
    -------
    intfunc: function object
             Correct integration function depending on length of x.

    fnargs: dictionary
            Dictionary of arguments to integration function.
    """
    if ispower2(len(x)-1):
        intfunc = integrate.romb
        fnargs = {"dx":x[1]-x[0]}
    elif len(x) > 0:
        intfunc = integrate.simps
        fnargs = {"x":x}
    else:
        raise ValueError("Cannot integrate length 0 array!")
    return intfunc, fnargs

def cartesian_product(lists, previous_elements = []):
    """Generator of cartesian products of lists."""
    if len(lists) == 1:
        for elem in lists[0]:
            yield previous_elements + [elem, ]
    else:
        for elem in lists[0]:
            for x in cartesian_product(lists[1:], previous_elements + [elem, ]):
                yield x

def cartesian(arrays, out=None):
    """
    Generate a cartesian product of input arrays.

    Parameters
    ----------
    arrays : list of array-like
        1-D arrays to form the cartesian product of.
    out : ndarray
        Array to place the cartesian product in.

    Returns
    -------
    out : ndarray
        2-D array of shape (M, len(arrays)) containing cartesian products
        formed of input arrays.

    Examples
    --------
    >>> cartesian(([1, 2, 3], [4, 5], [6, 7]))
    array([[1, 4, 6],
           [1, 4, 7],
           [1, 5, 6],
           [1, 5, 7],
           [2, 4, 6],
           [2, 4, 7],
           [2, 5, 6],
           [2, 5, 7],
           [3, 4, 6],
           [3, 4, 7],
           [3, 5, 6],
           [3, 5, 7]])

    """

    arrays = [np.asarray(x) for x in arrays]
    dtype = arrays[0].dtype

    n = np.prod([x.size for x in arrays])
    if out is None:
        out = np.zeros([n, len(arrays)], dtype=dtype)

    m = n / arrays[0].size
    out[:,0] = np.repeat(arrays[0], m)
    if arrays[1:]:
        cartesian(arrays[1:], out=out[0:m,1:])
        for j in xrange(1, arrays[0].size):
            out[j*m:(j+1)*m,1:] = out[0:m,1:]
    return out

def ensurepath(path):
    """Check that the path for given directory exists and create it if not."""
    #Does path exist?
    if not os.path.isdir(os.path.dirname(path)):
        try:
            os.makedirs(os.path.dirname(path))
        except OSError:
            raise OSError("Error creating results directory!")

def seq(min=0.0, max=None, inc=1.0, type=float,
        return_type='NumPyArray'):
    """
    Generate numbers from min to (and including!) max,
    with increment of inc. Safe alternative to arange.
    The return_type string governs the type of the returned
    sequence of numbers ('NumPyArray', 'list', or 'tuple').
    """
    if max is None: # allow sequence(3) to be 0., 1., 2., 3.
        # take 1st arg as max, min as 0, and inc=1
        max = min; min = 0.0; inc = 1.0
    r = np.arange(min, max + inc/2.0, inc, type)
    if return_type == 'NumPyArray' or return_type == np.ndarray:
        return r
    elif return_type == 'list':
        return r.tolist()
    elif return_type == 'tuple':
        return tuple(r.tolist())
    return

def find_nearest(array,value):
    """
    Find the index of the number in `array` which is nearest to `value`.
    """
    idx=(np.abs(array-value)).argmin()
    return array[idx]

def find_nearest_ix(array,value):
    """
    Find the index of the number in `array` which is nearest to `value`.
    """
    return (np.abs(array-value)).argmin()

def startlogging(log, logfile, loglevel=logging.INFO, consolelevel=None):
    """Start the logging system to store rotational file based log."""

    try:
        from cloghandler import ConcurrentRotatingFileHandler as RFHandler
    except ImportError:
    # Next 2 lines are optional:  issue a warning to the user
        from warnings import warn
        warn("ConcurrentLogHandler package not installed.  Using builtin log handler")
        from logging.handlers import RotatingFileHandler as RFHandler

    if not consolelevel:
        consolelevel = loglevel

    log.setLevel(loglevel)
    #create file handler and set level to debug
    fh = RFHandler(filename=logfile, maxBytes=2**20, backupCount=50)
    fh.setLevel(loglevel)
    #create console handler and set level to error
    ch = logging.StreamHandler()
    ch.setLevel(consolelevel)
    #create formatter
    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
    #add formatter to fh
    fh.setFormatter(formatter)
    #add formatter to ch
    ch.setFormatter(formatter)
    #add fh to logger
    log.addHandler(fh)
    #add ch to logger
    log.addHandler(ch)
    log.debug("Logging started at level %d", loglevel)
    return log

<commit_msg>Use absolute path in helpers.ensurepath.<commit_after># -*- coding: utf-8 -*-
"""Helper functions by Ian Huston
    $Id: helpers.py,v 1.19 2010/01/18 16:50:34 ith Exp $

    Provides helper functions for use elsewhere"""

from __future__ import division # Get rid of integer division problems, i.e. 1/2=0
import numpy as np
import re
from scipy import integrate
import os
import os.path
import logging

def nanfillstart(a, l):
    """Return an array of length l by appending array a to end of block of NaNs along axis 0."""
    if len(a) >= l:
        return a #Array already as long or longer than required
    else:
        bshape = np.array(a.shape)
        bshape[0] = l - bshape[0]
        b = np.ones(bshape)*np.NaN
        c = np.concatenate((b,a))
        return c

def eto10(number):
    """Convert scientific notation e.g. 1e-5 to 1x10^{-5} for use in LaTeX, converting to string."""
    s = re.sub(r'e(\S\d+)', r'\\times 10^{\1}', str(number))
    return s

def klegend(ks,mpc=False):
    """Return list of string representations of k modes for legend."""
    klist = []
    for k in ks:
        if mpc:
            str = r"$k=" + eto10(k) + r"M_{\mathrm{PL}} = " + eto10(mpl2invmpc(k)) + r" M\mathrm{pc}$"
        else:
            str = r"$k=" + eto10(k) + r"M_{\mathrm{PL}}$"
        klist.append(str)
    return klist

def invmpc2mpl(x=1):
    """Convert from Mpc^-1 to Mpl (reduced Planck Mass)"""
    return 2.625e-57*x

def mpl2invmpc(x=1):
    """Convert from Mpl (reduced Planck Mass) to Mpc^-1"""
    return 3.8095e+56*x

def ispower2(n):
    """Returns the log base 2 of n if n is a power of 2, zero otherwise.

    Note the potential ambiguity if n==1: 2**0==1, interpret accordingly."""

    bin_n = np.binary_repr(n)[1:]
    if '1' in bin_n:
        return 0
    else:
        return len(bin_n)

def removedups(l):
    """Return an array with duplicates removed but order retained.

    An array is returned no matter what the input type. The first of each duplicate is retained.

    Parameters
    ----------
    l: array_like
       Array (or list etc.) of values with duplicates that are to be removed.

    Returns
    -------
    retlist: ndarray
             Array of values with duplicates removed but order intact.
    """
    retlist = np.array([])
    for x in l:
        if x not in retlist:
            retlist = np.append(retlist, x)
    return retlist

def getintfunc(x):
    """Return the correct function to integrate with.

    Checks the given set of values and returns either scipy.integrate.romb
    or scipy.integrate.simps. This depends on whether the number of values is a
    power of 2 + 1 as required by romb.

    Parameters
    ----------
    x: array_like
       Array of x values to check

    Returns
    -------
    intfunc: function object
             Correct integration function depending on length of x.

    fnargs: dictionary
            Dictionary of arguments to integration function.
    """
    if ispower2(len(x)-1):
        intfunc = integrate.romb
        fnargs = {"dx":x[1]-x[0]}
    elif len(x) > 0:
        intfunc = integrate.simps
        fnargs = {"x":x}
    else:
        raise ValueError("Cannot integrate length 0 array!")
    return intfunc, fnargs

def cartesian_product(lists, previous_elements = []):
    """Generator of cartesian products of lists."""
    if len(lists) == 1:
        for elem in lists[0]:
            yield previous_elements + [elem, ]
    else:
        for elem in lists[0]:
            for x in cartesian_product(lists[1:], previous_elements + [elem, ]):
                yield x

def cartesian(arrays, out=None):
    """
    Generate a cartesian product of input arrays.

    Parameters
    ----------
    arrays : list of array-like
        1-D arrays to form the cartesian product of.
    out : ndarray
        Array to place the cartesian product in.

    Returns
    -------
    out : ndarray
        2-D array of shape (M, len(arrays)) containing cartesian products
        formed of input arrays.

    Examples
    --------
    >>> cartesian(([1, 2, 3], [4, 5], [6, 7]))
    array([[1, 4, 6],
           [1, 4, 7],
           [1, 5, 6],
           [1, 5, 7],
           [2, 4, 6],
           [2, 4, 7],
           [2, 5, 6],
           [2, 5, 7],
           [3, 4, 6],
           [3, 4, 7],
           [3, 5, 6],
           [3, 5, 7]])

    """

    arrays = [np.asarray(x) for x in arrays]
    dtype = arrays[0].dtype

    n = np.prod([x.size for x in arrays])
    if out is None:
        out = np.zeros([n, len(arrays)], dtype=dtype)

    m = n / arrays[0].size
    out[:,0] = np.repeat(arrays[0], m)
    if arrays[1:]:
        cartesian(arrays[1:], out=out[0:m,1:])
        for j in xrange(1, arrays[0].size):
            out[j*m:(j+1)*m,1:] = out[0:m,1:]
    return out

def ensurepath(path):
    """Check that the path for given directory exists and create it if not."""

    #Get absolute path
    path = os.path.abspath(path)
    #Does path exist?
    if not os.path.isdir(os.path.dirname(path)):
        try:
            os.makedirs(os.path.dirname(path))
        except OSError:
            raise OSError("Error creating results directory!")

def seq(min=0.0, max=None, inc=1.0, type=float,
        return_type='NumPyArray'):
    """
    Generate numbers from min to (and including!) max,
    with increment of inc. Safe alternative to arange.
    The return_type string governs the type of the returned
    sequence of numbers ('NumPyArray', 'list', or 'tuple').
    """
    if max is None: # allow sequence(3) to be 0., 1., 2., 3.
        # take 1st arg as max, min as 0, and inc=1
        max = min; min = 0.0; inc = 1.0
    r = np.arange(min, max + inc/2.0, inc, type)
    if return_type == 'NumPyArray' or return_type == np.ndarray:
        return r
    elif return_type == 'list':
        return r.tolist()
    elif return_type == 'tuple':
        return tuple(r.tolist())
    return

def find_nearest(array,value):
    """
    Find the index of the number in `array` which is nearest to `value`.
    """
    idx=(np.abs(array-value)).argmin()
    return array[idx]

def find_nearest_ix(array,value):
    """
    Find the index of the number in `array` which is nearest to `value`.
    """
    return (np.abs(array-value)).argmin()

def startlogging(log, logfile, loglevel=logging.INFO, consolelevel=None):
    """Start the logging system to store rotational file based log."""

    try:
        from cloghandler import ConcurrentRotatingFileHandler as RFHandler
    except ImportError:
    # Next 2 lines are optional:  issue a warning to the user
        from warnings import warn
        warn("ConcurrentLogHandler package not installed.  Using builtin log handler")
        from logging.handlers import RotatingFileHandler as RFHandler

    if not consolelevel:
        consolelevel = loglevel

    log.setLevel(loglevel)
    #create file handler and set level to debug
    fh = RFHandler(filename=logfile, maxBytes=2**20, backupCount=50)
    fh.setLevel(loglevel)
    #create console handler and set level to error
    ch = logging.StreamHandler()
    ch.setLevel(consolelevel)
    #create formatter
    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
    #add formatter to fh
    fh.setFormatter(formatter)
    #add formatter to ch
    ch.setFormatter(formatter)
    #add fh to logger
    log.addHandler(fh)
    #add ch to logger
    log.addHandler(ch)
    log.debug("Logging started at level %d", loglevel)
    return log

<|endoftext|>


<commit_before># --------------------------------------------------------------------------------------
# Copyright 2016, Benedikt J. Daurer, Filipe R.N.C. Maia, Max F. Hantke, Carl Nettelblad
# Hummingbird is distributed under the terms of the Simplified BSD License.
# -------------------------------------------------------------------------
import numpy
import utils.io
from backend.record import add_record

def patterson(evt, type, key, mask=None, threshold=None, diameter_pix=None):
    """TODO: missing docstring

    .. note:: This feature depends on the python package `libspimage <https://github.com/FilipeMaia/libspimage>`_.
    """
    success, module = utils.io.load_spimage()
    if not success:
        print "Skipping analysis.patterson.patterson"
        return
    img  = evt[type][key].data
    if mask is None:
        mask = numpy.ones(shape=img.shape, dtype="bool")
    else:
        mask = numpy.array(mask, dtype="bool")

    P = spimage.patterson(img, mask, floor_cut=100., mask_smooth=4., darkfield_x=None, darkfield_y=None, darkfield_sigma=None, normalize_median=True, radial_boost=False, log_boost=True, gauss_damp=True, gauss_damp_sigma=None, gauss_damp_threshold=None, subtract_fourier_kernel=True, log_min=1., full_output=False)
    v = evt["analysis"]
    add_record(v, "analysis", "patterson", abs(P), unit='')

    if threshold is not None:
        M = P > threshold
        if diameter_pix is not None:
            Y,X = numpy.indices(P.shape)
            X -= P.shape[1]/2
            Y -= P.shape[0]/2
            Rsq = X**2+Y**2
            M *= Rsq > diameter_pix**2
        multiple_score = M.sum()
        add_record(v, "analysis", "multiple score", multiple_score, unit='')

<commit_msg>Patterson module name change<commit_after># --------------------------------------------------------------------------------------
# Copyright 2016, Benedikt J. Daurer, Filipe R.N.C. Maia, Max F. Hantke, Carl Nettelblad
# Hummingbird is distributed under the terms of the Simplified BSD License.
# -------------------------------------------------------------------------
import numpy
import utils.io
from backend.record import add_record

def patterson(evt, type, key, mask=None, threshold=None, diameter_pix=None):
    """TODO: missing docstring

    .. note:: This feature depends on the python package `libspimage <https://github.com/FilipeMaia/libspimage>`_.
    """
    success, module = utils.io.load_spimage()
    if not success:
        print "Skipping analysis.patterson.patterson"
        return
    img  = evt[type][key].data
    if mask is None:
        mask = numpy.ones(shape=img.shape, dtype="bool")
    else:
        mask = numpy.array(mask, dtype="bool")

    P = module.patterson(img, mask, floor_cut=100., mask_smooth=4., darkfield_x=None, darkfield_y=None, darkfield_sigma=None, normalize_median=True, radial_boost=False, log_boost=True, gauss_damp=True, gauss_damp_sigma=None, gauss_damp_threshold=None, subtract_fourier_kernel=True, log_min=1., full_output=False)
    v = evt["analysis"]
    add_record(v, "analysis", "patterson", abs(P), unit='')

    if threshold is not None:
        M = P > threshold
        if diameter_pix is not None:
            Y,X = numpy.indices(P.shape)
            X -= P.shape[1]/2
            Y -= P.shape[0]/2
            Rsq = X**2+Y**2
            M *= Rsq > diameter_pix**2
        multiple_score = M.sum()
        add_record(v, "analysis", "multiple score", multiple_score, unit='')

<|endoftext|>


<commit_before>import math
import gdal
import numpy as np

from tornado.httpclient import AsyncHTTPClient
from skimage.draw import line
from tornado.gen import Return
from tornado import gen

ZOOM=12

class CoordSystem(object):
    @classmethod
    def lnglat_to_pixel(cls, lnglat, zoom=ZOOM):
        lng, lat = lnglat
        lat *= math.pi / 180.0
        lng *= math.pi / 180.0
        x = 128.0 / math.pi * 2**zoom * (lng + math.pi)
        y = 128.0 / math.pi * 2**zoom * (math.pi - math.log(math.tan(math.pi / 4.0 + lat / 2.0)))
        #print lnglat, '->', (x,y), '->', (x//256,y//256)
        return round(x), round(y)

    @classmethod
    def lnglat_to_tile(cls, lnglat, zoom=ZOOM):
        r = cls.lnglat_to_pixel(lnglat, zoom)
        return cls.pixel_to_tile(r, zoom)

    @classmethod
    def pixel_to_tile(cls, pixel, zoom=ZOOM):
        return (int(pixel[0]//256), 2**zoom - int(pixel[1]//256) - 1)

    @classmethod
    def pixel_to_lnglat(cls, point, zoom=ZOOM):
        x, y = point
        lat = (4.0 * math.atan(math.exp(math.pi - y * math.pi / (128.0 * 2**zoom))) - math.pi) / 2.0
        lng = x * math.pi / (128.0 * 2**zoom) - math.pi
        lat *= 180.0 / math.pi
        lng *= 180.0 / math.pi
        return lng, lat

def load_float32_image(buffer):
    try:
        gdal.FileFromMemBuffer('/vsimem/temp', buffer)
        ds = gdal.Open('/vsimem/temp')
        channel = ds.GetRasterBand(1).ReadAsArray()
        ds = None #cleanup
        gdal.Unlink('/vsimem/temp') #cleanup
        return channel
    except Exception, e:
        ds = None #cleanup
        gdal.Unlink('/vsimem/temp') #cleanup
        raise e

class Tile(object):
    def __init__(self, zoom, pixel, url_template):
        self.zoom = zoom
        self.url_template = url_template
        self.pixel = (pixel[0]//255*255, pixel[1]//255*255) #round pixel to top left corner
        self.data = self._retrieve_data() #data is actually a future

    @property
    def url(self):
        tile = CoordSystem.pixel_to_tile(self.pixel, self.zoom)
        return self.url_template.format(z=self.zoom, x=tile[0], y=tile[1])

    @gen.coroutine
    def _retrieve_data(self):
        self.res = yield AsyncHTTPClient().fetch(self.url) #NOTE: currently will throw an http error if status code is not 200
        raise Return(load_float32_image(self.res.body) if self.res.code == 200 else None)

class TileSampler(object):
    """Samples tile values. Everything is in global pixel space at specified zoom"""

    def __init__(self, zoom=ZOOM, url_template='http://127.0.0.1:8080/{z}/{x}/{y}.tiff'):
        self.zoom = zoom
        self.url_template = url_template
        self._tiles = {}

    def _unique_rows(self, data):
        """Returns only the unique rows in a numpy array

        Args:
            data (array): numpy array to dedupe

        Returns:
            array: deduped numpy array
        """
        uniq = np.unique(data.view(data.dtype.descr * data.shape[1]))
        return uniq.view(data.dtype).reshape(-1, data.shape[1])

    @gen.coroutine
    def _sample_tile_pixels(self, tile_pixel, pixels):
        """Returns pixel's values which intersect a single tile"""
        xs = pixels[:,0]
        ys = pixels[:,1]
        xs = xs-tile_pixel[0]
        xs = xs[(xs[:]>=0) & (xs[:]<=255)] #filter to just this tile
        ys = ys-tile_pixel[1]
        ys = ys[(ys[:]>=0) & (ys[:]<=255)]
        tile_data = yield self.get_tile(tile_pixel).data
        raise Return(tile_data[ys.astype(int), xs.astype(int)]) #numpy is row column

    def get_tile(self, pixel):
        """Returns a tile. If tile already exists a cached version will be returned.

        Args:
            pixel ((int, int)): A pixel in the tile to be returned
        """
        tile_pixel = (pixel[0]//256*256, pixel[1]//256*256)
        return self._tiles.get(tile_pixel, Tile(self.zoom, tile_pixel, self.url_template))

    @gen.coroutine
    def sample_pixels(self, pixels):
        """Samples arbitrary pixel values from the map. Returned order may not match input order (will for lines)

        Args:
            pixels (array): 2d numpy array where each row is a pixel to sample

        Returns:
            array: numpy array of values
        """
        #determin required tiles
        tile_pixels = np.floor_divide(pixels, 256)*256
        tile_pixels = self._unique_rows(tile_pixels)
        #preload tiles
        for tile_pixel in tile_pixels: self.get_tile(tile_pixel)
        #sample tiles
        data = [(yield self._sample_tile_pixels(tile_pixel, pixels)) for tile_pixel in tile_pixels]
        raise Return(np.concatenate(data))

    @gen.coroutine
    def sample_line(self, pixel1, pixel2):
        """Samples a line of pixel values in the map. Pixel values should be ordered from pixel1 to pixel2.

        Args:
            pixel1 ((int,int)): the 1st coordinate of the line in global pixel space
            pixel2 ((int,int)): the 2nd coordinate of the line in global pixel space

        Returns:
            array: numpy array of values
        """
        xs, ys = line(pixel1[0], pixel1[1], pixel2[0], pixel2[1])
        pixels = np.dstack((xs, ys))[0]
        raise Return((yield self.sample_pixels(pixels)))

    @gen.coroutine
    def sample_pixel(self, pixel):
        raise Return((yield self.sample_pixels(np.array([pixel])))[0])

#currently /12/655/2515.tiff
#should be /12/656/2514.tiff<commit_msg>fixed<commit_after>import math
import gdal
import numpy as np

from tornado.httpclient import AsyncHTTPClient
from skimage.draw import line
from tornado.gen import Return
from tornado import gen

ZOOM=12

class CoordSystem(object):
    @classmethod
    def lnglat_to_pixel(cls, lnglat, zoom=ZOOM):
        lng, lat = lnglat
        lat *= math.pi / 180.0
        lng *= math.pi / 180.0
        x = 128.0 / math.pi * 2**zoom * (lng + math.pi)
        y = 128.0 / math.pi * 2**zoom * (math.pi - math.log(math.tan(math.pi / 4.0 + lat / 2.0)))
        #print lnglat, '->', (x,y), '->', (x//256,y//256)
        return round(x), round(y)

    @classmethod
    def lnglat_to_tile(cls, lnglat, zoom=ZOOM):
        r = cls.lnglat_to_pixel(lnglat, zoom)
        return cls.pixel_to_tile(r, zoom)

    @classmethod
    def pixel_to_tile(cls, pixel, zoom=ZOOM):
        return (int(pixel[0]//256), 2**zoom - int(pixel[1]//256) - 1)

    @classmethod
    def pixel_to_lnglat(cls, point, zoom=ZOOM):
        x, y = point
        lat = (4.0 * math.atan(math.exp(math.pi - y * math.pi / (128.0 * 2**zoom))) - math.pi) / 2.0
        lng = x * math.pi / (128.0 * 2**zoom) - math.pi
        lat *= 180.0 / math.pi
        lng *= 180.0 / math.pi
        return lng, lat

def load_float32_image(buffer):
    try:
        gdal.FileFromMemBuffer('/vsimem/temp', buffer)
        ds = gdal.Open('/vsimem/temp')
        channel = ds.GetRasterBand(1).ReadAsArray()
        ds = None #cleanup
        gdal.Unlink('/vsimem/temp') #cleanup
        return channel
    except Exception, e:
        ds = None #cleanup
        gdal.Unlink('/vsimem/temp') #cleanup
        raise e

class Tile(object):
    def __init__(self, zoom, pixel, url_template):
        self.zoom = zoom
        self.url_template = url_template
        self.pixel = (pixel[0]//256*256, pixel[1]//256*256) #round pixel to top left corner
        self.data = self._retrieve_data() #data is actually a future

    @property
    def url(self):
        tile = CoordSystem.pixel_to_tile(self.pixel, self.zoom)
        return self.url_template.format(z=self.zoom, x=tile[0], y=tile[1])

    @gen.coroutine
    def _retrieve_data(self):
        self.res = yield AsyncHTTPClient().fetch(self.url) #NOTE: currently will throw an http error if status code is not 200
        raise Return(load_float32_image(self.res.body) if self.res.code == 200 else None)

class TileSampler(object):
    """Samples tile values. Everything is in global pixel space at specified zoom"""

    def __init__(self, zoom=ZOOM, url_template='http://127.0.0.1:8080/{z}/{x}/{y}.tiff'):
        self.zoom = zoom
        self.url_template = url_template
        self._tiles = {}

    def _unique_rows(self, data):
        """Returns only the unique rows in a numpy array

        Args:
            data (array): numpy array to dedupe

        Returns:
            array: deduped numpy array
        """
        uniq = np.unique(data.view(data.dtype.descr * data.shape[1]))
        return uniq.view(data.dtype).reshape(-1, data.shape[1])

    @gen.coroutine
    def _sample_tile_pixels(self, tile_pixel, pixels):
        """Returns pixel's values which intersect a single tile"""
        xs = pixels[:,0]
        ys = pixels[:,1]
        xs = xs-tile_pixel[0]
        xs = xs[(xs[:]>=0) & (xs[:]<=255)] #filter to just this tile
        ys = ys-tile_pixel[1]
        ys = ys[(ys[:]>=0) & (ys[:]<=255)]
        tile_data = yield self.get_tile(tile_pixel).data
        raise Return(tile_data[ys.astype(int), xs.astype(int)]) #numpy is row column

    def get_tile(self, pixel):
        """Returns a tile. If tile already exists a cached version will be returned.

        Args:
            pixel ((int, int)): A pixel in the tile to be returned
        """
        tile_pixel = (pixel[0]//256*256, pixel[1]//256*256)
        return self._tiles.get(tile_pixel, Tile(self.zoom, tile_pixel, self.url_template))

    @gen.coroutine
    def sample_pixels(self, pixels):
        """Samples arbitrary pixel values from the map. Returned order may not match input order (will for lines)

        Args:
            pixels (array): 2d numpy array where each row is a pixel to sample

        Returns:
            array: numpy array of values
        """
        #determin required tiles
        tile_pixels = np.floor_divide(pixels, 256)*256
        tile_pixels = self._unique_rows(tile_pixels)
        #preload tiles
        for tile_pixel in tile_pixels: self.get_tile(tile_pixel)
        #sample tiles
        data = [(yield self._sample_tile_pixels(tile_pixel, pixels)) for tile_pixel in tile_pixels]
        raise Return(np.concatenate(data))

    @gen.coroutine
    def sample_line(self, pixel1, pixel2):
        """Samples a line of pixel values in the map. Pixel values should be ordered from pixel1 to pixel2.

        Args:
            pixel1 ((int,int)): the 1st coordinate of the line in global pixel space
            pixel2 ((int,int)): the 2nd coordinate of the line in global pixel space

        Returns:
            array: numpy array of values
        """
        xs, ys = line(pixel1[0], pixel1[1], pixel2[0], pixel2[1])
        pixels = np.dstack((xs, ys))[0]
        raise Return((yield self.sample_pixels(pixels)))

    @gen.coroutine
    def sample_pixel(self, pixel):
        raise Return((yield self.sample_pixels(np.array([pixel])))[0])

#currently /12/655/2515.tiff
#should be /12/656/2514.tiff<|endoftext|>


<commit_before>#Copyright (c) 2015 noteness
#Copyright (c) 2015 Jesús "JeDa" Hernández
#Permission is hereby granted, free of charge, to any person obtaining a copy
#of this software and associated documentation files (the "Software"), to deal
#in the Software without restriction, including without limitation the rights
#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#copies of the Software, and to permit persons to whom the Software is
#furnished to do so, subject to the following conditions:

#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
#SOFTWARE.

import sys
py_version = sys.version_info
major = int(py_version [0])
if major == 2:
    from __future__ import print_function
    from __future__ import unicode_literals

EAT_PLUGIN = 1
EAT_HEXCHAT = 1
EAT_ALL = 1
EAT_NONE = 1
PRI_HIGHEST = 1
PRI_HIGH = 1
PRI_NORM = 1
PRI_LOW = 1
PRI_LOWEST = 1

if __name__ == "__main__":
    import os
    import sys
    broken = 0
    print("HexChat addons test script.")
    print("There might be weird stuff. Don't worry. :)\n")
    for val in os.listdir("addons"):
        if val.endswith('__'):
            continue
        if val.endswith('.pyc'):
            continue
        val = val.replace(".py", "")
        print("Testing {0}".format(val))
        try:
            __import__('addons.{0}'.format(val), globals=globals())
            print("{0} is WORKING.".format(val))
        except Exception as err:
            print("{0} is FAILING. ({1})".format(val, err))
            broken = 1
    if broken == 1:
        print("\nThere are broken addons. :(")
        sys.exit(1)
    else:
        print("\nEverything is fine. :)")
        sys.exit(0)

def hook_command(name, function, help):
    function(help.split(" "), [help, help.split(" ")[1]], None)

def command(command):
    split = command.replace(unicode("\x034",encoding='utf8'), "").split(" ")
    if split[0] == "say":
        print("<testuser> {0}".format(command.replace(split[0] + " ", "")))
    elif split[0] == "me":
        print("* testuser {0}".format(command.replace(split[0] + " ", "")))
    elif split[0] == "msg" or split[0] == "privmsg":
        print(">{0}< {1}".format(split[1], command.replace(split[0] + " ", "").replace(split[1] + " ", "")))
    elif split[0] == "notice":
        print("->{0}<- {1}".format(split[1], command.replace(split[0] + " ", "").replace(split[1] + " ", "")))

testword = [':Slavetator!noteness@unaffiliated/nessessary129/bot/slavetator', 'NICK', ':Slavetator___']
testwordeol = [':Slavetator!noteness@unaffiliated/nessessary129/bot/slavetator NICK :Slavetator___', 'NICK :Slavetator___', ':Slavetator___']
testdata = None
def hook_server(raw,func,priority):
    if raw == 'NICK':
        func(testword,testwordeol,testdata)
def prnt(stri):
    print(stri)

<commit_msg>Specify encoding<commit_after># -*- coding: utf8 -*-
#Copyright (c) 2015 noteness
#Copyright (c) 2015 Jesús "JeDa" Hernández
#Permission is hereby granted, free of charge, to any person obtaining a copy
#of this software and associated documentation files (the "Software"), to deal
#in the Software without restriction, including without limitation the rights
#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#copies of the Software, and to permit persons to whom the Software is
#furnished to do so, subject to the following conditions:

#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
#SOFTWARE.

EAT_PLUGIN = 1
EAT_HEXCHAT = 1
EAT_ALL = 1
EAT_NONE = 1
PRI_HIGHEST = 1
PRI_HIGH = 1
PRI_NORM = 1
PRI_LOW = 1
PRI_LOWEST = 1

if __name__ == "__main__":
    import os
    import sys
    broken = 0
    print("HexChat addons test script.")
    print("There might be weird stuff. Don't worry. :)\n")
    for val in os.listdir("addons"):
        if val.endswith('__'):
            continue
        if val.endswith('.pyc'):
            continue
        val = val.replace(".py", "")
        print("Testing {0}".format(val))
        try:
            __import__('addons.{0}'.format(val), globals=globals())
            print("{0} is WORKING.".format(val))
        except Exception as err:
            print("{0} is FAILING. ({1})".format(val, err))
            broken = 1
    if broken == 1:
        print("\nThere are broken addons. :(")
        sys.exit(1)
    else:
        print("\nEverything is fine. :)")
        sys.exit(0)

def hook_command(name, function, help):
    function(help.split(" "), [help, help.split(" ")[1]], None)

def command(command):
    split = command.replace(unicode("\x034",encoding='utf8'), "").split(" ")
    if split[0] == "say":
        print("<testuser> {0}".format(command.replace(split[0] + " ", "")))
    elif split[0] == "me":
        print("* testuser {0}".format(command.replace(split[0] + " ", "")))
    elif split[0] == "msg" or split[0] == "privmsg":
        print(">{0}< {1}".format(split[1], command.replace(split[0] + " ", "").replace(split[1] + " ", "")))
    elif split[0] == "notice":
        print("->{0}<- {1}".format(split[1], command.replace(split[0] + " ", "").replace(split[1] + " ", "")))

testword = [':Slavetator!noteness@unaffiliated/nessessary129/bot/slavetator', 'NICK', ':Slavetator___']
testwordeol = [':Slavetator!noteness@unaffiliated/nessessary129/bot/slavetator NICK :Slavetator___', 'NICK :Slavetator___', ':Slavetator___']
testdata = None
def hook_server(raw,func,priority):
    if raw == 'NICK':
        func(testword,testwordeol,testdata)
def prnt(stri):
    print(stri)

<|endoftext|>