<commit_before># coding=utf-8
from __future__ import absolute_import
from __future__ import unicode_literals

import ghdiff
import six

import io
from django.contrib import messages
from django.utils.translation import ugettext as _

from corehq.apps.app_manager.exceptions import (
    FormNotFoundException,
    ModuleNotFoundException,
)
from corehq.apps.hqwebapp.tasks import send_html_email_async
from corehq.apps.translations.app_translations.utils import (
    BulkAppTranslationUpdater,
    get_unicode_dicts,
    is_form_sheet,
    is_module_sheet,
    is_modules_and_forms_sheet,
    is_single_sheet,
    get_menu_or_form_by_sheet_name,
    get_menu_or_form_by_unique_id,
)
from corehq.apps.translations.const import MODULES_AND_FORMS_SHEET_NAME
from corehq.apps.translations.app_translations.upload_form import BulkAppTranslationFormUpdater
from corehq.apps.translations.app_translations.upload_module import BulkAppTranslationModuleUpdater
from corehq.apps.translations.exceptions import BulkAppTranslationsException

def validate_bulk_app_translation_upload(app, workbook, email, lang_to_compare):
    from corehq.apps.translations.validator import UploadedTranslationsValidator
    msgs = UploadedTranslationsValidator(app, workbook, lang_to_compare).compare()
    if msgs:
        _email_app_translations_discrepancies(msgs, email, app.name)
        return [(messages.error, _("Issues found. You should receive an email shortly."))]
    else:
        return [(messages.success, _("No issues found."))]

def _email_app_translations_discrepancies(msgs, email, app_name):
    html_content = ghdiff.default_css
    for sheet_name, msg in msgs.items():
        html_content += "{}".format(sheet_name) + msg

    subject = _("App Translations Discrepancies for {}").format(app_name)
    text_content = _("Hi, PFA file for discrepancies found for app translations.")
    html_attachment = {
        'title': "{} Discrepancies.html".format(app_name),
        'file_obj': io.StringIO(html_content),
        'mimetype': 'text/html',
    }
    send_html_email_async.delay(subject, email, text_content, file_attachments=[html_attachment])

def process_bulk_app_translation_upload(app, workbook, expected_headers, lang=None):
    """
    Process the bulk upload file for the given app.
    We return these message tuples instead of calling them now to allow this
    function to be used independently of request objects.

    :return: Returns a list of message tuples. The first item in each tuple is
    a function like django.contrib.messages.error, and the second is a string.
    """
    msgs = []
    error = _check_for_workbook_error(app, workbook, expected_headers)
    if error:
        msgs.append((messages.error, error))
        return msgs

    processed_sheets = set()
    sheet_name_to_unique_id = {}
    for sheet in workbook.worksheets:
        try:
            _check_for_sheet_error(app, sheet, expected_headers, processed_sheets=processed_sheets)
        except BulkAppTranslationsException as e:
            msgs.append((messages.error, six.text_type(e)))
            continue

        processed_sheets.add(sheet.worksheet.title)

        warnings = _check_for_sheet_warnings(app, sheet, expected_headers)
        for warning in warnings:
            msgs.append((messages.warning, warning))

        if is_single_sheet(sheet.worksheet.title):
            msgs.extend(_process_single_sheet(app, sheet, names_map=sheet_name_to_unique_id, lang=lang))
        else:
            msgs.extend(_process_rows(app, sheet.worksheet.title, sheet, names_map=sheet_name_to_unique_id))

    msgs.append(
        (messages.success, _("App Translations Updated!"))
    )
    return msgs

def _process_single_sheet(app, sheet, names_map, lang=None):
    """
    A single-sheet translation file deals with only one language, and
    fits all the items to be translated onto the same sheet. All items
    share the same columns. If the column is not applicable to the row,
    it is left empty.

    :param app: The application being translated
    :param sheet: The worksheet containing the translations
    :param names_map: A map of sheet_name (like "menu1" or "menu1_form1") to
                      module/form unique_id, used to fetch a module/form
                      even if it has been moved since the worksheet was created
    :param lang: The language that the app is being translated into
    :return: A list of error messages or an empty list
    """
    msgs = []
    module_or_form = None
    # Rows pertaining to a module (e.g. case list / case detail) or a form
    # (i.e. questions). Keyed on that module/form's identifier:
    module_or_form_rows = {}
    # Rows about the app's modules and forms, like their names and unique IDs:
    modules_and_forms_rows = []
    rows = []
    for row in sheet:
        if not row['case_property'] and not row['list_or_detail'] and not row['label']:
            modules_and_forms_rows.append(row)
        elif module_or_form != row['menu_or_form']:
            module_or_form_rows[module_or_form] = rows
            module_or_form = row['menu_or_form']
            rows = [row]
        else:
            rows.append(row)
    module_or_form_rows[module_or_form] = rows
    # Process modules_and_forms_rows first to populate names_map with their unique IDs
    msgs.extend(_process_rows(app, MODULES_AND_FORMS_SHEET_NAME,
                              modules_and_forms_rows, names_map, lang=lang))
    # Then process the rows for the modules and the forms.
    for module_or_form, rows in six.iteritems(module_or_form_rows):
        msgs.extend(_process_rows(app, module_or_form, rows, names_map, lang=lang))
    return msgs

def _process_rows(app, sheet_name, rows, names_map, lang=None):
    """
    Processes the rows of a worksheet of translations.

    This is the complement of get_bulk_app_sheets_by_name() and
    get_bulk_app_single_sheet_by_name(), from
    corehq/apps/translations/app_translations/download.py, which creates
    these worksheets and rows.

    :param app: The application being translated
    :param sheet_name: The tab name of the sheet being processed.
                       e.g. "menu1", "menu1_form1", or "Menus_and_forms"
    :param rows: The rows in the worksheet
    :param names_map: A map of sheet_name to module/form unique_id, used
                      to fetch a module/form even if it has been moved
                      since the worksheet was created
    :param lang: The language that the app is being translated into
    :return: A list of error messages or an empty list
    """
    if not sheet_name or not rows:
        return []

    if is_modules_and_forms_sheet(sheet_name):
        updater = BulkAppTranslationModulesAndFormsUpdater(app, names_map, lang=lang)
        return updater.update(rows)

    if is_module_sheet(sheet_name):
        unique_id = names_map.get(sheet_name)
        try:
            updater = BulkAppTranslationModuleUpdater(app, sheet_name, unique_id, lang=lang)
        except ModuleNotFoundException:
            return [(
                messages.error,
                _('Invalid menu in row "%s", skipping row.') % sheet_name
            )]
        return updater.update(rows)

    if is_form_sheet(sheet_name):
        unique_id = names_map.get(sheet_name)
        try:
            updater = BulkAppTranslationFormUpdater(app, sheet_name, unique_id, lang=lang)
        except FormNotFoundException:
            return [(
                messages.error,
                _('Invalid form in row "%s", skipping row.') % sheet_name
            )]
        return updater.update(rows)

    return [(
        messages.error,
        _('Did not recognize "%s", skipping row.') % sheet_name
    )]

def _check_for_workbook_error(app, workbook, headers):
    if len(headers) == 1 and len(workbook.worksheets) > 1:
        return _("Expected a single sheet. If you are uploading a multi-sheet file, "
                 "please select 'All Languages'.")
    if len(headers) > 1 and len(workbook.worksheets) == 1:
        return _("File contains only one sheet. If you are uploading a single-language file, "
                 "please select a language.")

def _check_for_sheet_error(app, sheet, headers, processed_sheets=Ellipsis):
    expected_sheets = {h[0]: h[1] for h in headers}

    if sheet.worksheet.title in processed_sheets:
        raise BulkAppTranslationsException(_('Sheet "%s" was repeated. Only the first occurrence has been '
                                             'processed.') % sheet.worksheet.title)

    expected_headers = expected_sheets.get(sheet.worksheet.title, None)
    if expected_headers is None:
        raise BulkAppTranslationsException(_('Skipping sheet "%s", could not recognize title') %
                                           sheet.worksheet.title)

    num_required_headers = 0
    if is_modules_and_forms_sheet(sheet.worksheet.title):
        num_required_headers = 1    # type
    elif is_module_sheet(sheet.worksheet.title):
        num_required_headers = 2    # case property, list or detail
    elif is_form_sheet(sheet.worksheet.title):
        num_required_headers = 1    # label
    elif is_single_sheet(sheet.worksheet.title):
        num_required_headers = 4    # menu or form, case property, list or detail, label

    expected_required_headers = tuple(expected_headers[:num_required_headers])
    actual_required_headers = tuple(sheet.headers[:num_required_headers])
    if expected_required_headers != actual_required_headers:
        raise BulkAppTranslationsException(_('Skipping sheet {title}: expected first columns to be '
                                             '{expected}').format(
                                                 title=sheet.worksheet.title,
                                                 expected=", ".join(expected_required_headers)))

def _check_for_sheet_warnings(app, sheet, headers):
    warnings = []
    expected_sheets = {h[0]: h[1] for h in headers}
    expected_headers = expected_sheets.get(sheet.worksheet.title, None)

    missing_cols = _get_missing_cols(app, sheet, headers)
    extra_cols = set(sheet.headers) - set(expected_headers)

    if len(missing_cols) > 0:
        warnings.append((_('Sheet "%s" has fewer columns than expected. '
            'Sheet will be processed but the following translations will be unchanged: %s')
            % (sheet.worksheet.title, ", ".join(missing_cols))))

    if len(extra_cols) > 0:
        warnings.append(_('Sheet "%s" has unrecognized columns. '
            'Sheet will be processed but ignoring the following columns: %s')
            % (sheet.worksheet.title, ", ".join(extra_cols)))

    return warnings

def _get_missing_cols(app, sheet, headers):
    expected_sheets = {h[0]: h[1] for h in headers}
    expected_headers = expected_sheets.get(sheet.worksheet.title, None)
    return set(expected_columns) - set(sheet.headers)

class BulkAppTranslationModulesAndFormsUpdater(BulkAppTranslationUpdater):
    def __init__(self, app, names_map, lang=None):
        super(BulkAppTranslationModulesAndFormsUpdater, self).__init__(app, lang)
        self.sheet_name_to_unique_id = names_map

    def update(self, rows):
        """
        This handles updating module/form names and menu media
        (the contents of the "Menus and forms" sheet in the multi-tab upload).
        """
        self.msgs = []
        for row in get_unicode_dicts(rows):
            sheet_name = row.get('menu_or_form', '')
            # The unique_id column is populated on the "Menus_and_forms" sheet in multi-sheet translation files,
            # and in the "name / menu media" row in single-sheet translation files.
            unique_id = row.get('unique_id')

            if unique_id and sheet_name not in self.sheet_name_to_unique_id:
                # If we have a value for unique_id, save it in self.sheet_name_to_unique_id so we can look it up
                # for rows where the unique_id column is not populated.
                self.sheet_name_to_unique_id[sheet_name] = unique_id
            elif not unique_id and sheet_name in self.sheet_name_to_unique_id:
                # If we don't have a value for unique_id, try to fetch it from self.sheet_name_to_unique_id
                unique_id = self.sheet_name_to_unique_id[sheet_name]

            try:
                if unique_id:
                    document = get_menu_or_form_by_unique_id(self.app, unique_id, sheet_name)
                else:
                    document = get_menu_or_form_by_sheet_name(self.app, sheet_name)
            except (ModuleNotFoundException, FormNotFoundException, ValueError) as err:
                self.msgs.append((messages.error, six.text_type(err)))
                continue

            self.update_translation_dict('default_', document.name, row)

            # Update menu media
            for lang in self.langs:
                image_header = 'image_%s' % lang
                if image_header in row:
                    document.set_icon(lang, row[image_header])

                audio_header = 'audio_%s' % lang
                if audio_header in row:
                    document.set_audio(lang, row[audio_header])

        return self.msgs
<commit_msg>Inlined _get_missing_cols<commit_after># coding=utf-8
from __future__ import absolute_import
from __future__ import unicode_literals

import ghdiff
import six

import io
from django.contrib import messages
from django.utils.translation import ugettext as _

from corehq.apps.app_manager.exceptions import (
    FormNotFoundException,
    ModuleNotFoundException,
)
from corehq.apps.hqwebapp.tasks import send_html_email_async
from corehq.apps.translations.app_translations.utils import (
    BulkAppTranslationUpdater,
    get_unicode_dicts,
    is_form_sheet,
    is_module_sheet,
    is_modules_and_forms_sheet,
    is_single_sheet,
    get_menu_or_form_by_sheet_name,
    get_menu_or_form_by_unique_id,
)
from corehq.apps.translations.const import MODULES_AND_FORMS_SHEET_NAME
from corehq.apps.translations.app_translations.upload_form import BulkAppTranslationFormUpdater
from corehq.apps.translations.app_translations.upload_module import BulkAppTranslationModuleUpdater
from corehq.apps.translations.exceptions import BulkAppTranslationsException

def validate_bulk_app_translation_upload(app, workbook, email, lang_to_compare):
    from corehq.apps.translations.validator import UploadedTranslationsValidator
    msgs = UploadedTranslationsValidator(app, workbook, lang_to_compare).compare()
    if msgs:
        _email_app_translations_discrepancies(msgs, email, app.name)
        return [(messages.error, _("Issues found. You should receive an email shortly."))]
    else:
        return [(messages.success, _("No issues found."))]

def _email_app_translations_discrepancies(msgs, email, app_name):
    html_content = ghdiff.default_css
    for sheet_name, msg in msgs.items():
        html_content += "{}".format(sheet_name) + msg

    subject = _("App Translations Discrepancies for {}").format(app_name)
    text_content = _("Hi, PFA file for discrepancies found for app translations.")
    html_attachment = {
        'title': "{} Discrepancies.html".format(app_name),
        'file_obj': io.StringIO(html_content),
        'mimetype': 'text/html',
    }
    send_html_email_async.delay(subject, email, text_content, file_attachments=[html_attachment])

def process_bulk_app_translation_upload(app, workbook, expected_headers, lang=None):
    """
    Process the bulk upload file for the given app.
    We return these message tuples instead of calling them now to allow this
    function to be used independently of request objects.

    :return: Returns a list of message tuples. The first item in each tuple is
    a function like django.contrib.messages.error, and the second is a string.
    """
    msgs = []
    error = _check_for_workbook_error(app, workbook, expected_headers)
    if error:
        msgs.append((messages.error, error))
        return msgs

    processed_sheets = set()
    sheet_name_to_unique_id = {}
    for sheet in workbook.worksheets:
        try:
            _check_for_sheet_error(app, sheet, expected_headers, processed_sheets=processed_sheets)
        except BulkAppTranslationsException as e:
            msgs.append((messages.error, six.text_type(e)))
            continue

        processed_sheets.add(sheet.worksheet.title)

        warnings = _check_for_sheet_warnings(app, sheet, expected_headers)
        for warning in warnings:
            msgs.append((messages.warning, warning))

        if is_single_sheet(sheet.worksheet.title):
            msgs.extend(_process_single_sheet(app, sheet, names_map=sheet_name_to_unique_id, lang=lang))
        else:
            msgs.extend(_process_rows(app, sheet.worksheet.title, sheet, names_map=sheet_name_to_unique_id))

    msgs.append(
        (messages.success, _("App Translations Updated!"))
    )
    return msgs

def _process_single_sheet(app, sheet, names_map, lang=None):
    """
    A single-sheet translation file deals with only one language, and
    fits all the items to be translated onto the same sheet. All items
    share the same columns. If the column is not applicable to the row,
    it is left empty.

    :param app: The application being translated
    :param sheet: The worksheet containing the translations
    :param names_map: A map of sheet_name (like "menu1" or "menu1_form1") to
                      module/form unique_id, used to fetch a module/form
                      even if it has been moved since the worksheet was created
    :param lang: The language that the app is being translated into
    :return: A list of error messages or an empty list
    """
    msgs = []
    module_or_form = None
    # Rows pertaining to a module (e.g. case list / case detail) or a form
    # (i.e. questions). Keyed on that module/form's identifier:
    module_or_form_rows = {}
    # Rows about the app's modules and forms, like their names and unique IDs:
    modules_and_forms_rows = []
    rows = []
    for row in sheet:
        if not row['case_property'] and not row['list_or_detail'] and not row['label']:
            modules_and_forms_rows.append(row)
        elif module_or_form != row['menu_or_form']:
            module_or_form_rows[module_or_form] = rows
            module_or_form = row['menu_or_form']
            rows = [row]
        else:
            rows.append(row)
    module_or_form_rows[module_or_form] = rows
    # Process modules_and_forms_rows first to populate names_map with their unique IDs
    msgs.extend(_process_rows(app, MODULES_AND_FORMS_SHEET_NAME,
                              modules_and_forms_rows, names_map, lang=lang))
    # Then process the rows for the modules and the forms.
    for module_or_form, rows in six.iteritems(module_or_form_rows):
        msgs.extend(_process_rows(app, module_or_form, rows, names_map, lang=lang))
    return msgs

def _process_rows(app, sheet_name, rows, names_map, lang=None):
    """
    Processes the rows of a worksheet of translations.

    This is the complement of get_bulk_app_sheets_by_name() and
    get_bulk_app_single_sheet_by_name(), from
    corehq/apps/translations/app_translations/download.py, which creates
    these worksheets and rows.

    :param app: The application being translated
    :param sheet_name: The tab name of the sheet being processed.
                       e.g. "menu1", "menu1_form1", or "Menus_and_forms"
    :param rows: The rows in the worksheet
    :param names_map: A map of sheet_name to module/form unique_id, used
                      to fetch a module/form even if it has been moved
                      since the worksheet was created
    :param lang: The language that the app is being translated into
    :return: A list of error messages or an empty list
    """
    if not sheet_name or not rows:
        return []

    if is_modules_and_forms_sheet(sheet_name):
        updater = BulkAppTranslationModulesAndFormsUpdater(app, names_map, lang=lang)
        return updater.update(rows)

    if is_module_sheet(sheet_name):
        unique_id = names_map.get(sheet_name)
        try:
            updater = BulkAppTranslationModuleUpdater(app, sheet_name, unique_id, lang=lang)
        except ModuleNotFoundException:
            return [(
                messages.error,
                _('Invalid menu in row "%s", skipping row.') % sheet_name
            )]
        return updater.update(rows)

    if is_form_sheet(sheet_name):
        unique_id = names_map.get(sheet_name)
        try:
            updater = BulkAppTranslationFormUpdater(app, sheet_name, unique_id, lang=lang)
        except FormNotFoundException:
            return [(
                messages.error,
                _('Invalid form in row "%s", skipping row.') % sheet_name
            )]
        return updater.update(rows)

    return [(
        messages.error,
        _('Did not recognize "%s", skipping row.') % sheet_name
    )]

def _check_for_workbook_error(app, workbook, headers):
    if len(headers) == 1 and len(workbook.worksheets) > 1:
        return _("Expected a single sheet. If you are uploading a multi-sheet file, "
                 "please select 'All Languages'.")
    if len(headers) > 1 and len(workbook.worksheets) == 1:
        return _("File contains only one sheet. If you are uploading a single-language file, "
                 "please select a language.")

def _check_for_sheet_error(app, sheet, headers, processed_sheets=Ellipsis):
    expected_sheets = {h[0]: h[1] for h in headers}

    if sheet.worksheet.title in processed_sheets:
        raise BulkAppTranslationsException(_('Sheet "%s" was repeated. Only the first occurrence has been '
                                             'processed.') % sheet.worksheet.title)

    expected_headers = expected_sheets.get(sheet.worksheet.title, None)
    if expected_headers is None:
        raise BulkAppTranslationsException(_('Skipping sheet "%s", could not recognize title') %
                                           sheet.worksheet.title)

    num_required_headers = 0
    if is_modules_and_forms_sheet(sheet.worksheet.title):
        num_required_headers = 1    # type
    elif is_module_sheet(sheet.worksheet.title):
        num_required_headers = 2    # case property, list or detail
    elif is_form_sheet(sheet.worksheet.title):
        num_required_headers = 1    # label
    elif is_single_sheet(sheet.worksheet.title):
        num_required_headers = 4    # menu or form, case property, list or detail, label

    expected_required_headers = tuple(expected_headers[:num_required_headers])
    actual_required_headers = tuple(sheet.headers[:num_required_headers])
    if expected_required_headers != actual_required_headers:
        raise BulkAppTranslationsException(_('Skipping sheet {title}: expected first columns to be '
                                             '{expected}').format(
                                                 title=sheet.worksheet.title,
                                                 expected=", ".join(expected_required_headers)))

def _check_for_sheet_warnings(app, sheet, headers):
    warnings = []
    expected_sheets = {h[0]: h[1] for h in headers}
    expected_headers = expected_sheets.get(sheet.worksheet.title, None)

    missing_cols = set(expected_headers) - set(sheet.headers)
    extra_cols = set(sheet.headers) - set(expected_headers)

    if len(missing_cols) > 0:
        warnings.append((_('Sheet "%s" has fewer columns than expected. '
            'Sheet will be processed but the following translations will be unchanged: %s')
            % (sheet.worksheet.title, ", ".join(missing_cols))))

    if len(extra_cols) > 0:
        warnings.append(_('Sheet "%s" has unrecognized columns. '
            'Sheet will be processed but ignoring the following columns: %s')
            % (sheet.worksheet.title, ", ".join(extra_cols)))

    return warnings

class BulkAppTranslationModulesAndFormsUpdater(BulkAppTranslationUpdater):
    def __init__(self, app, names_map, lang=None):
        super(BulkAppTranslationModulesAndFormsUpdater, self).__init__(app, lang)
        self.sheet_name_to_unique_id = names_map

    def update(self, rows):
        """
        This handles updating module/form names and menu media
        (the contents of the "Menus and forms" sheet in the multi-tab upload).
        """
        self.msgs = []
        for row in get_unicode_dicts(rows):
            sheet_name = row.get('menu_or_form', '')
            # The unique_id column is populated on the "Menus_and_forms" sheet in multi-sheet translation files,
            # and in the "name / menu media" row in single-sheet translation files.
            unique_id = row.get('unique_id')

            if unique_id and sheet_name not in self.sheet_name_to_unique_id:
                # If we have a value for unique_id, save it in self.sheet_name_to_unique_id so we can look it up
                # for rows where the unique_id column is not populated.
                self.sheet_name_to_unique_id[sheet_name] = unique_id
            elif not unique_id and sheet_name in self.sheet_name_to_unique_id:
                # If we don't have a value for unique_id, try to fetch it from self.sheet_name_to_unique_id
                unique_id = self.sheet_name_to_unique_id[sheet_name]

            try:
                if unique_id:
                    document = get_menu_or_form_by_unique_id(self.app, unique_id, sheet_name)
                else:
                    document = get_menu_or_form_by_sheet_name(self.app, sheet_name)
            except (ModuleNotFoundException, FormNotFoundException, ValueError) as err:
                self.msgs.append((messages.error, six.text_type(err)))
                continue

            self.update_translation_dict('default_', document.name, row)

            # Update menu media
            for lang in self.langs:
                image_header = 'image_%s' % lang
                if image_header in row:
                    document.set_icon(lang, row[image_header])

                audio_header = 'audio_%s' % lang
                if audio_header in row:
                    document.set_audio(lang, row[audio_header])

        return self.msgs
<|endoftext|>


<commit_before>from django.core.management.base import BaseCommand
from django.core.mail import mail_admins

from haystack import site

from antxetamedia.multimedia.models import Media

class Command(BaseCommand):
    args = 'None'
    help = 'Synchronize all Medias that are not'

    def handle(self, *args, **kwargs):
        for media in Media.objects.order_by('-pk').filter(
                synchronize=True, is_synchronized=False).iterator():
            if media.local:
                print 'Syncing {0}...'.format(media.related_object or media),
                try:
                    media.sync()
                except Exception, msg:
                    print 'Failed: {0}'.format(msg)
                else:
                    if media.is_synchronized:
                        print 'OK'
                    else:
                        print 'Failed'
                    media.save()
<commit_msg>Get rid of unneeded import<commit_after>from django.core.management.base import BaseCommand
from django.core.mail import mail_admins

from antxetamedia.multimedia.models import Media

class Command(BaseCommand):
    args = 'None'
    help = 'Synchronize all Medias that are not'

    def handle(self, *args, **kwargs):
        for media in Media.objects.order_by('-pk').filter(
                synchronize=True, is_synchronized=False).iterator():
            if media.local:
                print 'Syncing {0}...'.format(media.related_object or media),
                try:
                    media.sync()
                except Exception, msg:
                    print 'Failed: {0}'.format(msg)
                else:
                    if media.is_synchronized:
                        print 'OK'
                    else:
                        print 'Failed'
                    media.save()
<|endoftext|>


<commit_before>VERSION = '0.9.6'
<commit_msg>paypal version bump to 0.9.7<commit_after>VERSION = '0.9.7'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright(C) 2010  Romain Bignon
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

import datetime
import time

from .base import IBaseCap, CapBaseObject, NotLoaded

__all__ = ['ICapMessages', 'ICapMessagesPost', 'Message', 'Thread', 'CantSendMessage']

class Message(CapBaseObject):
    IS_HTML = 0x001          # The content is HTML formatted
    IS_UNREAD = 0x002        # The message is unread
    IS_ACCUSED = 0x004       # The receiver has read this message
    IS_NOT_ACCUSED = 0x008   # The receiver has not read this message

    def __init__(self, thread, id,
                       title,
                       sender,
                       receiver=NotLoaded,
                       date=None,
                       parent=NotLoaded,
                       content=NotLoaded,
                       signature=NotLoaded,
                       children=NotLoaded,
                       flags=0):
        CapBaseObject.__init__(self, id)
        self.thread = thread
        self.title = title
        self.sender = sender
        self.receiver = receiver

        if date is None:
            date = datetime.datetime.utcnow()
        self.date = date

        if isinstance(parent, Message):
            self.parent = parent
        else:
            self.parent = NotLoaded
            self._parent_id = parent

        self.content = content
        self.signature = signature
        self.children = children
        self.flags = flags

    @property
    def date_int(self):
        return int(time.strftime('%Y%m%d%H%M%S', self.date.timetuple()))

    @property
    def full_id(self):
        return '%s.%s' % (self.thread.id, self.id)

    @property
    def full_parent_id(self):
        if self.parent:
            return self.parent.full_id()
        elif self._parent_id is None:
            return ''
        elif self._parent_id is NotLoaded:
            return NotLoaded
        else:
            return '%s.%s' % (self.thread.id, self._parent_id)

    def __eq__(self, msg):
        return self.thread.id == msg.thread.id and self.id == msg.id

    def __repr__(self):
        result = '<Message id="%s" title="%s" date="%s" from="%s">' % (
            self.full_id, self.title, self.date, self.sender)
        return result.encode('utf-8')

class Thread(CapBaseObject):
    def __init__(self, id):
        CapBaseObject.__init__(self, id)
        self.root = NotLoaded
        self.title = NotLoaded
        self.date = NotLoaded
        self.nb_messages = NotLoaded
        self.nb_unread = NotLoaded

    def iter_all_messages(self):
        if self.root:
            yield self.root
            for m in self._iter_all_messages(self.root):
                yield m

    def _iter_all_messages(self, message):
        if message.children:
            for child in message.children:
                yield child
                for m in self._iter_all_messages(child):
                    yield m

class ICapMessages(IBaseCap):
    def iter_threads(self):
        """
        Iterates on threads, from newers to olders.

        @return [iter]  Thread objects
        """
        raise NotImplementedError()

    def get_thread(self, id):
        """
        Get a specific thread.

        @return [Thread]  the Thread object
        """
        raise NotImplementedError()

    def iter_unread_messages(self, thread=None):
        """
        Iterates on messages which hasn't been marked as read.

        @param thread  thread name (optional)
        @return [iter]  Message objects
        """
        raise NotImplementedError()

    def set_message_read(self, message):
        """
        Set a message as read.

        @param [message]  message read (or ID)
        """
        raise NotImplementedError()

class CantSendMessage(Exception):
    pass

class ICapMessagesPost(IBaseCap):
    def post_message(self, message):
        """
        Post a reply.

        @param message  Message object
        @return
        """
        raise NotImplementedError()
<commit_msg>title and sender are not needed anymore<commit_after># -*- coding: utf-8 -*-

# Copyright(C) 2010  Romain Bignon
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

import datetime
import time

from .base import IBaseCap, CapBaseObject, NotLoaded

__all__ = ['ICapMessages', 'ICapMessagesPost', 'Message', 'Thread', 'CantSendMessage']

class Message(CapBaseObject):
    IS_HTML = 0x001          # The content is HTML formatted
    IS_UNREAD = 0x002        # The message is unread
    IS_ACCUSED = 0x004       # The receiver has read this message
    IS_NOT_ACCUSED = 0x008   # The receiver has not read this message

    def __init__(self, thread, id,
                       title=NotLoaded,
                       sender=NotLoaded,
                       receiver=NotLoaded,
                       date=None,
                       parent=NotLoaded,
                       content=NotLoaded,
                       signature=NotLoaded,
                       children=NotLoaded,
                       flags=0):
        CapBaseObject.__init__(self, id)
        self.thread = thread
        self.title = title
        self.sender = sender
        self.receiver = receiver

        if date is None:
            date = datetime.datetime.utcnow()
        self.date = date

        if isinstance(parent, Message):
            self.parent = parent
        else:
            self.parent = NotLoaded
            self._parent_id = parent

        self.content = content
        self.signature = signature
        self.children = children
        self.flags = flags

    @property
    def date_int(self):
        return int(time.strftime('%Y%m%d%H%M%S', self.date.timetuple()))

    @property
    def full_id(self):
        return '%s.%s' % (self.thread.id, self.id)

    @property
    def full_parent_id(self):
        if self.parent:
            return self.parent.full_id()
        elif self._parent_id is None:
            return ''
        elif self._parent_id is NotLoaded:
            return NotLoaded
        else:
            return '%s.%s' % (self.thread.id, self._parent_id)

    def __eq__(self, msg):
        return self.thread.id == msg.thread.id and self.id == msg.id

    def __repr__(self):
        result = '<Message id="%s" title="%s" date="%s" from="%s">' % (
            self.full_id, self.title, self.date, self.sender)
        return result.encode('utf-8')

class Thread(CapBaseObject):
    def __init__(self, id):
        CapBaseObject.__init__(self, id)
        self.root = NotLoaded
        self.title = NotLoaded
        self.date = NotLoaded
        self.nb_messages = NotLoaded
        self.nb_unread = NotLoaded

    def iter_all_messages(self):
        if self.root:
            yield self.root
            for m in self._iter_all_messages(self.root):
                yield m

    def _iter_all_messages(self, message):
        if message.children:
            for child in message.children:
                yield child
                for m in self._iter_all_messages(child):
                    yield m

class ICapMessages(IBaseCap):
    def iter_threads(self):
        """
        Iterates on threads, from newers to olders.

        @return [iter]  Thread objects
        """
        raise NotImplementedError()

    def get_thread(self, id):
        """
        Get a specific thread.

        @return [Thread]  the Thread object
        """
        raise NotImplementedError()

    def iter_unread_messages(self, thread=None):
        """
        Iterates on messages which hasn't been marked as read.

        @param thread  thread name (optional)
        @return [iter]  Message objects
        """
        raise NotImplementedError()

    def set_message_read(self, message):
        """
        Set a message as read.

        @param [message]  message read (or ID)
        """
        raise NotImplementedError()

class CantSendMessage(Exception):
    pass

class ICapMessagesPost(IBaseCap):
    def post_message(self, message):
        """
        Post a reply.

        @param message  Message object
        @return
        """
        raise NotImplementedError()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function

"""phy main CLI tool.

Usage:

    phy --help

"""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import sys
import os.path as op
import argparse
from textwrap import dedent

import numpy as np

from ..ext.six import exec_

#------------------------------------------------------------------------------
# Parser utilities
#------------------------------------------------------------------------------

class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter,
                      argparse.RawDescriptionHelpFormatter):
    pass

class Parser(argparse.ArgumentParser):
    def error(self, message):
        sys.stderr.write(message + '\n\n')
        self.print_help()
        sys.exit(2)

_examples = dedent("""

examples:
  phy -v                display the version of phy
  phy describe my_file.kwik
                        display information about a Kwik dataset
  phy spikesort my_params.prm
                        run the whole suite (spike detection and clustering)
  phy detect my_params.prm
                        run spike detection on a parameters file
  phy cluster-auto my_file.kwik --num-clusters-start=100
                        run klustakwik on a dataset
  phy cluster-manual my_file.kwik
                        run the manual clustering GUI

""")

#------------------------------------------------------------------------------
# Parser creator
#------------------------------------------------------------------------------

class ParserCreator(object):
    def __init__(self):
        self.create_main()
        self.create_describe()
        self.create_traces()
        self.create_spikesort()
        self.create_detect()
        self.create_auto()
        self.create_manual()
        self.create_notebook()

    @property
    def parser(self):
        return self._parser

    def _add_sub_parser(self, name, desc):
        p = self._subparsers.add_parser(name, help=desc, description=desc)
        self._add_options(p)
        return p

    def _add_options(self, parser):
        parser.add_argument('--debug', '-d',
                            action='store_true',
                            help='activate debug logging mode')

        parser.add_argument('--profiler', '-p',
                            action='store_true',
                            help='activate the profiler')

        parser.add_argument('--line-profiler', '-lp',
                            dest='line_profiler',
                            action='store_true',
                            help='activate the line-profiler -- you '
                                 'need to decorate the functions '
                                 'to profile with `@profile` '
                                 'in the code')

        parser.add_argument('--ipython', '-i', action='store_true',
                            help='launch the script in an interactive '
                            'IPython console')

    def create_main(self):
        import phy

        desc = sys.modules['phy'].__doc__
        self._parser = Parser(description=desc,
                              epilog=_examples,
                              formatter_class=CustomFormatter,
                              )
        self._parser.set_defaults(func=None)
        self._parser.add_argument('--version', '-v',
                                  action='version',
                                  version=phy.__version__,
                                  help='print the version of phy')
        self._add_options(self._parser)
        self._subparsers = self._parser.add_subparsers(dest='command',
                                                       title='subcommand',
                                                       )

    def create_describe(self):
        desc = 'describe a `.kwik` file'
        p = self._add_sub_parser('describe', desc)
        p.add_argument('file', help='path to a `.kwik` file')
        p.add_argument('--clustering', default='main',
                       help='name of the clustering to use')
        p.set_defaults(func=describe)

    def create_traces(self):
        desc = 'show the traces of a raw data file'
        p = self._add_sub_parser('traces', desc)
        p.add_argument('file', help='path to a `.kwd` or `.dat` file')
        p.add_argument('interval', help='interval in number '
                       'of samples',)
        p.add_argument('--n-channels', '-n',
                       help='number of channels in the recording '
                       '(only required when using a flat binary file)')
        p.add_argument('--dtype',
                       help='NumPy data type '
                       '(only required when using a flat binary file)',
                       default='int16',
                       )
        p.set_defaults(func=traces)

    def create_spikesort(self):
        desc = 'launch the whole spike sorting pipeline on a `.prm` file'
        p = self._add_sub_parser('spikesort', desc)
        p.add_argument('file', help='path to a `.prm` file')
        p.add_argument('--kwik-path', help='filename of the `.kwik` file '
                       'to create (by default, `"experiment_name".kwik`)')
        p.add_argument('--overwrite', action='store_true', default=False,
                       help='overwrite the `.kwik` file ')
        p.set_defaults(func=spikesort)

    def create_detect(self):
        desc = 'launch the spike detection algorithm on a `.prm` file'
        p = self._add_sub_parser('detect', desc)
        p.add_argument('file', help='path to a `.prm` file')
        p.add_argument('--kwik-path', help='filename of the `.kwik` file '
                       'to create (by default, `"experiment_name".kwik`)')
        p.add_argument('--overwrite', action='store_true', default=False,
                       help='overwrite the `.kwik` file ')
        p.set_defaults(func=detect)

    def create_manual(self):
        desc = 'launch the manual clustering GUI on a `.kwik` file'
        p = self._add_sub_parser('cluster-manual', desc)
        p.add_argument('file', help='path to a `.kwik` file')
        p.add_argument('--clustering', default='main',
                       help='name of the clustering to use')
        p.add_argument('--cluster-ids', '-c',
                       help='list of clusters to select initially')
        p.set_defaults(func=cluster_manual)

    def create_auto(self):
        desc = 'launch the automatic clustering algorithm on a `.kwik` file'
        p = self._add_sub_parser('cluster-auto', desc)
        p.add_argument('file', help='path to a `.kwik` file')
        p.add_argument('num_starting_clusters', type=int,
                       help='initial number of clusters',
                       )
        p.add_argument('--clustering', default='main',
                       help='name of the clustering to use')
        p.set_defaults(func=cluster_auto)

    def create_notebook(self):
        # TODO
        pass

    def parse(self, args):
        return self._parser.parse_args(args)

#------------------------------------------------------------------------------
# Subcommand functions
#------------------------------------------------------------------------------

def _get_kwik_path(args):
    kwik_path = args.file

    if not op.exists(kwik_path):
        print("The file `{}` doesn't exist.".format(kwik_path))
        return

    return kwik_path

def _create_session(args, **kwargs):
    from phy.cluster import Session
    kwik_path = _get_kwik_path(args)
    session = Session(kwik_path, **kwargs)
    return session

def describe(args):
    from phy.io.kwik import KwikModel
    path = _get_kwik_path(args)
    model = KwikModel(path, clustering=args.clustering)
    return 'model.describe()', dict(model=model)

def traces(args):
    from vispy.app import run
    from phy.plot.traces import TraceView
    from phy.io.h5 import open_h5
    from phy.io.traces import read_kwd, read_dat

    path = args.file
    if path.endswith('.kwd'):
        f = open_h5(args.file)
        traces = read_kwd(f)
    elif path.endswith(('.dat', '.bin')):
        if not args.n_channels:
            raise ValueError("Please specify `--n-channels`.")
        if not args.dtype:
            raise ValueError("Please specify `--dtype`.")
        n_channels = int(args.n_channels)
        dtype = np.dtype(args.dtype)
        traces = read_dat(path, dtype=dtype, n_channels=n_channels)

    start, end = map(int, args.interval.split('-'))

    c = TraceView(keys='interactive')
    c.visual.traces = .01 * traces[start:end, ...]
    c.show()
    run()

    return None, None

def detect(args):
    from phy.io import create_kwik

    assert args.file.endswith('.prm')
    kwik_path = args.kwik_path
    kwik_path = create_kwik(args.file, overwrite=args.overwrite,
                            kwik_path=kwik_path)
    # Create the session with the newly-created .kwik file.
    args.file = kwik_path
    session = _create_session(args, use_store=False)
    return 'session.detect()', dict(session=session)

def cluster_auto(args):
    session = _create_session(args, use_store=False)
    ns = dict(session=session,
              clustering=args.clustering,
              n_s_clusters=args.num_starting_clusters,
              )
    cmd = ('session.cluster(clustering=clustering, '
           'num_starting_clusters=n_s_clusters)')
    return (cmd, ns)

def spikesort(args):
    from phy.io import create_kwik

    assert args.file.endswith('.prm')
    kwik_path = args.kwik_path
    kwik_path = create_kwik(args.file, overwrite=args.overwrite,
                            kwik_path=kwik_path)
    # Create the session with the newly-created .kwik file.
    args.file = kwik_path
    session = _create_session(args, use_store=False)

    ns = dict(session=session,
              n_s_clusters=100,  # TODO: better handling of KK parameters
              )
    cmd = ('session.detect(); session.cluster('
           'num_starting_clusters=n_s_clusters)')
    return (cmd, ns)

def cluster_manual(args):
    session = _create_session(args, clustering=args.clustering)
    cluster_ids = (list(map(int, args.cluster_ids.split(',')))
                   if args.cluster_ids else None)

    session = _create_session(args)
    session.model.describe()

    from phy.gui import start_qt_app
    start_qt_app()

    gui = session.show_gui(cluster_ids=cluster_ids, show=False)
    print("\nPress `ctrl+h` to see the list of keyboard shortcuts.\n")
    return 'gui.show()', dict(session=session, gui=gui, requires_qt=True)

#------------------------------------------------------------------------------
# Main functions
#------------------------------------------------------------------------------

def main():

    p = ParserCreator()
    args = p.parse(sys.argv[1:])

    if args.profiler or args.line_profiler:
        from phy.utils.testing import _enable_profiler, _profile
        prof = _enable_profiler(args.line_profiler)
    else:
        prof = None

    import phy
    if args.debug:
        phy.debug()

    func = args.func
    if func is None:
        p.parser.print_help()
        return

    cmd, ns = func(args)
    if not cmd:
        return
    requires_qt = ns.pop('requires_qt', False)
    requires_vispy = ns.pop('requires_vispy', False)

    # Default variables in namespace.
    ns.update(phy=phy, path=args.file)
    if 'session' in ns:
        ns['model'] = ns['session'].model

    # Interactive mode with IPython.
    if args.ipython:
        print("\nStarting IPython...")
        from IPython import start_ipython
        args_ipy = ["-i", "-c='{}'".format(cmd)]
        if requires_qt or requires_vispy:
            # Activate Qt event loop integration with Qt.
            args_ipy += ["--gui=qt"]
        start_ipython(args_ipy, user_ns=ns)
    else:
        if not prof:
            exec_(cmd, {}, ns)
        else:
            _profile(prof, cmd, {}, ns)

        if requires_qt:
            # Launch the Qt app.
            from phy.gui import run_qt_app
            run_qt_app()
        elif requires_vispy:
            # Launch the VisPy Qt app.
            from vispy.app import use_app, run
            use_app('pyqt4')
            run()

#------------------------------------------------------------------------------
# Entry point
#------------------------------------------------------------------------------

if __name__ == '__main__':
    main()
<commit_msg>Added --interval option in phy spikesort tool.<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function

"""phy main CLI tool.

Usage:

    phy --help

"""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import sys
import os.path as op
import argparse
from textwrap import dedent

import numpy as np

from ..ext.six import exec_

#------------------------------------------------------------------------------
# Parser utilities
#------------------------------------------------------------------------------

class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter,
                      argparse.RawDescriptionHelpFormatter):
    pass

class Parser(argparse.ArgumentParser):
    def error(self, message):
        sys.stderr.write(message + '\n\n')
        self.print_help()
        sys.exit(2)

_examples = dedent("""

examples:
  phy -v                display the version of phy
  phy describe my_file.kwik
                        display information about a Kwik dataset
  phy spikesort my_params.prm
                        run the whole suite (spike detection and clustering)
  phy detect my_params.prm
                        run spike detection on a parameters file
  phy cluster-auto my_file.kwik --num-clusters-start=100
                        run klustakwik on a dataset
  phy cluster-manual my_file.kwik
                        run the manual clustering GUI

""")

#------------------------------------------------------------------------------
# Parser creator
#------------------------------------------------------------------------------

class ParserCreator(object):
    def __init__(self):
        self.create_main()
        self.create_describe()
        self.create_traces()
        self.create_spikesort()
        self.create_detect()
        self.create_auto()
        self.create_manual()
        self.create_notebook()

    @property
    def parser(self):
        return self._parser

    def _add_sub_parser(self, name, desc):
        p = self._subparsers.add_parser(name, help=desc, description=desc)
        self._add_options(p)
        return p

    def _add_options(self, parser):
        parser.add_argument('--debug', '-d',
                            action='store_true',
                            help='activate debug logging mode')

        parser.add_argument('--profiler', '-p',
                            action='store_true',
                            help='activate the profiler')

        parser.add_argument('--line-profiler', '-lp',
                            dest='line_profiler',
                            action='store_true',
                            help='activate the line-profiler -- you '
                                 'need to decorate the functions '
                                 'to profile with `@profile` '
                                 'in the code')

        parser.add_argument('--ipython', '-i', action='store_true',
                            help='launch the script in an interactive '
                            'IPython console')

    def create_main(self):
        import phy

        desc = sys.modules['phy'].__doc__
        self._parser = Parser(description=desc,
                              epilog=_examples,
                              formatter_class=CustomFormatter,
                              )
        self._parser.set_defaults(func=None)
        self._parser.add_argument('--version', '-v',
                                  action='version',
                                  version=phy.__version__,
                                  help='print the version of phy')
        self._add_options(self._parser)
        self._subparsers = self._parser.add_subparsers(dest='command',
                                                       title='subcommand',
                                                       )

    def create_describe(self):
        desc = 'describe a `.kwik` file'
        p = self._add_sub_parser('describe', desc)
        p.add_argument('file', help='path to a `.kwik` file')
        p.add_argument('--clustering', default='main',
                       help='name of the clustering to use')
        p.set_defaults(func=describe)

    def create_traces(self):
        desc = 'show the traces of a raw data file'
        p = self._add_sub_parser('traces', desc)
        p.add_argument('file', help='path to a `.kwd` or `.dat` file')
        p.add_argument('interval', help='interval in number '
                       'of samples',)
        p.add_argument('--n-channels', '-n',
                       help='number of channels in the recording '
                       '(only required when using a flat binary file)')
        p.add_argument('--dtype',
                       help='NumPy data type '
                       '(only required when using a flat binary file)',
                       default='int16',
                       )
        p.set_defaults(func=traces)

    def create_spikesort(self):
        desc = 'launch the whole spike sorting pipeline on a `.prm` file'
        p = self._add_sub_parser('spikesort', desc)
        p.add_argument('file', help='path to a `.prm` file')
        p.add_argument('--kwik-path', help='filename of the `.kwik` file '
                       'to create (by default, `"experiment_name".kwik`)')
        p.add_argument('--overwrite', action='store_true', default=False,
                       help='overwrite the `.kwik` file ')
        p.add_argument('--interval',
                       help='detection interval in seconds (e.g. `0,10`)')
        p.set_defaults(func=spikesort)

    def create_detect(self):
        desc = 'launch the spike detection algorithm on a `.prm` file'
        p = self._add_sub_parser('detect', desc)
        p.add_argument('file', help='path to a `.prm` file')
        p.add_argument('--kwik-path', help='filename of the `.kwik` file '
                       'to create (by default, `"experiment_name".kwik`)')
        p.add_argument('--overwrite', action='store_true', default=False,
                       help='overwrite the `.kwik` file ')
        p.set_defaults(func=detect)

    def create_manual(self):
        desc = 'launch the manual clustering GUI on a `.kwik` file'
        p = self._add_sub_parser('cluster-manual', desc)
        p.add_argument('file', help='path to a `.kwik` file')
        p.add_argument('--clustering', default='main',
                       help='name of the clustering to use')
        p.add_argument('--cluster-ids', '-c',
                       help='list of clusters to select initially')
        p.set_defaults(func=cluster_manual)

    def create_auto(self):
        desc = 'launch the automatic clustering algorithm on a `.kwik` file'
        p = self._add_sub_parser('cluster-auto', desc)
        p.add_argument('file', help='path to a `.kwik` file')
        p.add_argument('num_starting_clusters', type=int,
                       help='initial number of clusters',
                       )
        p.add_argument('--clustering', default='main',
                       help='name of the clustering to use')
        p.set_defaults(func=cluster_auto)

    def create_notebook(self):
        # TODO
        pass

    def parse(self, args):
        return self._parser.parse_args(args)

#------------------------------------------------------------------------------
# Subcommand functions
#------------------------------------------------------------------------------

def _get_kwik_path(args):
    kwik_path = args.file

    if not op.exists(kwik_path):
        print("The file `{}` doesn't exist.".format(kwik_path))
        return

    return kwik_path

def _create_session(args, **kwargs):
    from phy.cluster import Session
    kwik_path = _get_kwik_path(args)
    session = Session(kwik_path, **kwargs)
    return session

def describe(args):
    from phy.io.kwik import KwikModel
    path = _get_kwik_path(args)
    model = KwikModel(path, clustering=args.clustering)
    return 'model.describe()', dict(model=model)

def traces(args):
    from vispy.app import run
    from phy.plot.traces import TraceView
    from phy.io.h5 import open_h5
    from phy.io.traces import read_kwd, read_dat

    path = args.file
    if path.endswith('.kwd'):
        f = open_h5(args.file)
        traces = read_kwd(f)
    elif path.endswith(('.dat', '.bin')):
        if not args.n_channels:
            raise ValueError("Please specify `--n-channels`.")
        if not args.dtype:
            raise ValueError("Please specify `--dtype`.")
        n_channels = int(args.n_channels)
        dtype = np.dtype(args.dtype)
        traces = read_dat(path, dtype=dtype, n_channels=n_channels)

    start, end = map(int, args.interval.split('-'))

    c = TraceView(keys='interactive')
    c.visual.traces = .01 * traces[start:end, ...]
    c.show()
    run()

    return None, None

def detect(args):
    from phy.io import create_kwik

    assert args.file.endswith('.prm')
    kwik_path = args.kwik_path
    kwik_path = create_kwik(args.file,
                            overwrite=args.overwrite,
                            kwik_path=kwik_path)
    # Create the session with the newly-created .kwik file.
    args.file = kwik_path
    session = _create_session(args, use_store=False)
    return 'session.detect()', dict(session=session)

def cluster_auto(args):
    session = _create_session(args, use_store=False)
    ns = dict(session=session,
              clustering=args.clustering,
              n_s_clusters=args.num_starting_clusters,
              )
    cmd = ('session.cluster(clustering=clustering, '
           'num_starting_clusters=n_s_clusters)')
    return (cmd, ns)

def spikesort(args):
    from phy.io import create_kwik

    assert args.file.endswith('.prm')
    kwik_path = args.kwik_path
    kwik_path = create_kwik(args.file, overwrite=args.overwrite,
                            kwik_path=kwik_path)
    # Create the session with the newly-created .kwik file.
    args.file = kwik_path
    session = _create_session(args, use_store=False)

    interval = args.interval
    if interval is not None:
        interval = list(map(float, interval.split(',')))

    ns = dict(session=session,
              interval=interval,
              n_s_clusters=100,  # TODO: better handling of KK parameters
              )
    cmd = ('session.detect(interval=interval); session.cluster('
           'num_starting_clusters=n_s_clusters)')
    return (cmd, ns)

def cluster_manual(args):
    session = _create_session(args, clustering=args.clustering)
    cluster_ids = (list(map(int, args.cluster_ids.split(',')))
                   if args.cluster_ids else None)

    session = _create_session(args)
    session.model.describe()

    from phy.gui import start_qt_app
    start_qt_app()

    gui = session.show_gui(cluster_ids=cluster_ids, show=False)
    print("\nPress `ctrl+h` to see the list of keyboard shortcuts.\n")
    return 'gui.show()', dict(session=session, gui=gui, requires_qt=True)

#------------------------------------------------------------------------------
# Main functions
#------------------------------------------------------------------------------

def main():

    p = ParserCreator()
    args = p.parse(sys.argv[1:])

    if args.profiler or args.line_profiler:
        from phy.utils.testing import _enable_profiler, _profile
        prof = _enable_profiler(args.line_profiler)
    else:
        prof = None

    import phy
    if args.debug:
        phy.debug()

    func = args.func
    if func is None:
        p.parser.print_help()
        return

    cmd, ns = func(args)
    if not cmd:
        return
    requires_qt = ns.pop('requires_qt', False)
    requires_vispy = ns.pop('requires_vispy', False)

    # Default variables in namespace.
    ns.update(phy=phy, path=args.file)
    if 'session' in ns:
        ns['model'] = ns['session'].model

    # Interactive mode with IPython.
    if args.ipython:
        print("\nStarting IPython...")
        from IPython import start_ipython
        args_ipy = ["-i", "-c='{}'".format(cmd)]
        if requires_qt or requires_vispy:
            # Activate Qt event loop integration with Qt.
            args_ipy += ["--gui=qt"]
        start_ipython(args_ipy, user_ns=ns)
    else:
        if not prof:
            exec_(cmd, {}, ns)
        else:
            _profile(prof, cmd, {}, ns)

        if requires_qt:
            # Launch the Qt app.
            from phy.gui import run_qt_app
            run_qt_app()
        elif requires_vispy:
            # Launch the VisPy Qt app.
            from vispy.app import use_app, run
            use_app('pyqt4')
            run()

#------------------------------------------------------------------------------
# Entry point
#------------------------------------------------------------------------------

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os, sys, subprocess
from optparse import OptionParser

p0=os.path.dirname(__file__)
jsunhappypy = os.path.join(p0, "jsunhappy.py")

import jsunhappy
import pinpoint
import compareJIT

parser = OptionParser()
parser.disable_interspersed_args()
parser.add_option("--comparejit",
                  action = "store_true", dest = "useCompareJIT",
                  default = False,
                  help = "After running the fuzzer, run the FCM lines against the engine in two configurations and compare the output.")
parser.add_option("--fuzzjs",
                  action = "store", dest = "fuzzjs",
                  default = os.path.join(p0, "jsfunfuzz.js"),
                  help = "Which fuzzer to run (e.g. jsfunfuzz.js or regexpfuzz.js)")
parser.add_option("--repo",
                  action = "store", dest = "repo",
                  default = os.path.expanduser("~/tracemonkey/"),
                  help = "The hg repository (e.g. ~/tracemonkey/), for bisection")
options, args = parser.parse_args(sys.argv[1:])

timeout = int(args[0])
knownPath = os.path.expanduser(args[1])
engine = args[2]
engineFlags = args[3:]
runThis = [engine] + engineFlags + ["-e", "maxRunTime=" + str(timeout*(1000/2)), "-f", options.fuzzjs]

def showtail(filename):
    cmd = "tail -n 20 %s" % filename
    print cmd
    print ""
    os.system(cmd)
    print ""
    print ""

def many_timed_runs():
    iteration = 0
    while True:
        iteration += 1
        logPrefix = tempDir + os.sep + "w" + str(iteration)

        level = jsunhappy.jsfunfuzzLevel(runThis, timeout, knownPath, logPrefix)

        oklevel = jsunhappy.JS_KNOWN_CRASH
        if options.fuzzjs.find("jsfunfuzz") != -1:
            # Allow hangs. Allow abnormal exits in js shell (OOM) and xpcshell (bug 613142).
            oklevel = jsunhappy.JS_ABNORMAL_EXIT
        elif options.fuzzjs.find("regexpfuzz") != -1:
            # Allow hangs (bug ??????)
            oklevel = jsunhappy.JS_TIMED_OUT

        if level > oklevel:
            showtail(logPrefix + "-out")
            showtail(logPrefix + "-err")

            # splice jsfunfuzz.js with `grep FRC wN-out`
            filenameToReduce = logPrefix + "-reduced.js"
            [before, after] = fuzzSplice(open(options.fuzzjs))
            newfileLines = before + linesWith(open(logPrefix + "-out"), "FRC") + after
            writeLinesToFile(newfileLines, logPrefix + "-orig.js")
            writeLinesToFile(newfileLines, filenameToReduce)

            # Run Lithium and autobisect (make a reduced testcase and find a regression window)
            itest = [jsunhappypy, str(level), str(timeout), knownPath]
            alsoRunChar = (level > jsunhappy.JS_DID_NOT_FINISH)
            pinpoint.pinpoint(itest, logPrefix, engine, engineFlags, filenameToReduce, options.repo, alsoRunChar=alsoRunChar)

        else:
            if options.useCompareJIT and level == jsunhappy.JS_FINE:
                jitcomparelines = linesWith(open(logPrefix + "-out"), "FCM") + ["try{print(uneval(this));}catch(e){}"]
                jitcomparefilename = logPrefix + "-cj-in.js"
                writeLinesToFile(jitcomparelines, jitcomparefilename)
                compareJIT.compareJIT(engine, jitcomparefilename, logPrefix + "-cj", knownPath, options.repo, timeout, deleteBoring=True)
            os.remove(logPrefix + "-out")
            os.remove(logPrefix + "-err")
            if (os.path.exists(logPrefix + "-crash")):
                os.remove(logPrefix + "-crash")
            if (os.path.exists(logPrefix + "-vg.xml")):
                os.remove(logPrefix + "-vg.xml")
            if (os.path.exists(logPrefix + "-core")):
                os.remove(logPrefix + "-core")

def fuzzSplice(file):
    '''Returns the lines of a file, minus the ones between the two lines containing SPLICE'''
    before = []
    after = []
    for line in file:
        before.append(line)
        if line.find("SPLICE") != -1:
            break
    for line in file:
        if line.find("SPLICE") != -1:
            after.append(line)
            break
    for line in file:
        after.append(line)
    file.close()
    return [before, after]

def linesWith(file, searchFor):
    '''Returns the lines from a file that contain a given string'''
    matchingLines = []
    for line in file:
        if line.find(searchFor) != -1:
            matchingLines.append(line)
    file.close()
    return matchingLines

def writeLinesToFile(lines, filename):
      f = open(filename, "w")
      f.writelines(lines)
      f.close()

def createTempDir():
    global tempDir
    i = 1
    while 1:
        tempDir = "wtmp" + str(i)
        # To avoid race conditions, we use try/except instead of exists/create
        # Hopefully we don't get any errors other than "File exists" :)
        try:
            os.mkdir(tempDir)
            break
        except OSError, e:
            i += 1
    print tempDir + os.sep

createTempDir()
many_timed_runs()
<commit_msg>Replace os.system call in multi_timed_run.py with subprocess.call<commit_after>#!/usr/bin/env python

import os, sys, subprocess
from optparse import OptionParser

p0=os.path.dirname(__file__)
jsunhappypy = os.path.join(p0, "jsunhappy.py")

import jsunhappy
import pinpoint
import compareJIT

parser = OptionParser()
parser.disable_interspersed_args()
parser.add_option("--comparejit",
                  action = "store_true", dest = "useCompareJIT",
                  default = False,
                  help = "After running the fuzzer, run the FCM lines against the engine in two configurations and compare the output.")
parser.add_option("--fuzzjs",
                  action = "store", dest = "fuzzjs",
                  default = os.path.join(p0, "jsfunfuzz.js"),
                  help = "Which fuzzer to run (e.g. jsfunfuzz.js or regexpfuzz.js)")
parser.add_option("--repo",
                  action = "store", dest = "repo",
                  default = os.path.expanduser("~/tracemonkey/"),
                  help = "The hg repository (e.g. ~/tracemonkey/), for bisection")
options, args = parser.parse_args(sys.argv[1:])

timeout = int(args[0])
knownPath = os.path.expanduser(args[1])
engine = args[2]
engineFlags = args[3:]
runThis = [engine] + engineFlags + ["-e", "maxRunTime=" + str(timeout*(1000/2)), "-f", options.fuzzjs]

def showtail(filename):
    cmd = []
    cmd.append('tail')
    cmd.append('-n')
    cmd.append('20')
    cmd.append(filename)
    print ' '.join(cmd)
    print ""
    subprocess.call(cmd)
    print ""
    print ""

def many_timed_runs():
    iteration = 0
    while True:
        iteration += 1
        logPrefix = tempDir + os.sep + "w" + str(iteration)

        level = jsunhappy.jsfunfuzzLevel(runThis, timeout, knownPath, logPrefix)

        oklevel = jsunhappy.JS_KNOWN_CRASH
        if options.fuzzjs.find("jsfunfuzz") != -1:
            # Allow hangs. Allow abnormal exits in js shell (OOM) and xpcshell (bug 613142).
            oklevel = jsunhappy.JS_ABNORMAL_EXIT
        elif options.fuzzjs.find("regexpfuzz") != -1:
            # Allow hangs (bug ??????)
            oklevel = jsunhappy.JS_TIMED_OUT

        if level > oklevel:
            showtail(logPrefix + "-out")
            showtail(logPrefix + "-err")

            # splice jsfunfuzz.js with `grep FRC wN-out`
            filenameToReduce = logPrefix + "-reduced.js"
            [before, after] = fuzzSplice(open(options.fuzzjs))
            newfileLines = before + linesWith(open(logPrefix + "-out"), "FRC") + after
            writeLinesToFile(newfileLines, logPrefix + "-orig.js")
            writeLinesToFile(newfileLines, filenameToReduce)

            # Run Lithium and autobisect (make a reduced testcase and find a regression window)
            itest = [jsunhappypy, str(level), str(timeout), knownPath]
            alsoRunChar = (level > jsunhappy.JS_DID_NOT_FINISH)
            pinpoint.pinpoint(itest, logPrefix, engine, engineFlags, filenameToReduce, options.repo, alsoRunChar=alsoRunChar)

        else:
            if options.useCompareJIT and level == jsunhappy.JS_FINE:
                jitcomparelines = linesWith(open(logPrefix + "-out"), "FCM") + ["try{print(uneval(this));}catch(e){}"]
                jitcomparefilename = logPrefix + "-cj-in.js"
                writeLinesToFile(jitcomparelines, jitcomparefilename)
                compareJIT.compareJIT(engine, jitcomparefilename, logPrefix + "-cj", knownPath, options.repo, timeout, deleteBoring=True)
            os.remove(logPrefix + "-out")
            os.remove(logPrefix + "-err")
            if (os.path.exists(logPrefix + "-crash")):
                os.remove(logPrefix + "-crash")
            if (os.path.exists(logPrefix + "-vg.xml")):
                os.remove(logPrefix + "-vg.xml")
            if (os.path.exists(logPrefix + "-core")):
                os.remove(logPrefix + "-core")

def fuzzSplice(file):
    '''Returns the lines of a file, minus the ones between the two lines containing SPLICE'''
    before = []
    after = []
    for line in file:
        before.append(line)
        if line.find("SPLICE") != -1:
            break
    for line in file:
        if line.find("SPLICE") != -1:
            after.append(line)
            break
    for line in file:
        after.append(line)
    file.close()
    return [before, after]

def linesWith(file, searchFor):
    '''Returns the lines from a file that contain a given string'''
    matchingLines = []
    for line in file:
        if line.find(searchFor) != -1:
            matchingLines.append(line)
    file.close()
    return matchingLines

def writeLinesToFile(lines, filename):
      f = open(filename, "w")
      f.writelines(lines)
      f.close()

def createTempDir():
    global tempDir
    i = 1
    while 1:
        tempDir = "wtmp" + str(i)
        # To avoid race conditions, we use try/except instead of exists/create
        # Hopefully we don't get any errors other than "File exists" :)
        try:
            os.mkdir(tempDir)
            break
        except OSError, e:
            i += 1
    print tempDir + os.sep

createTempDir()
many_timed_runs()
<|endoftext|>


<commit_before># Copyright (c) 2006, Neville-Neil Consulting
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# Neither the name of Neville-Neil Consulting nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# File: $Id: arp.py,v 1.2 2006/08/01 13:35:58 gnn Exp $
#
# Author: George V. Neville-Neil
#
# Description:  ARP packet class
import pcs
import struct
from socket import AF_INET, inet_ntop, inet_ntoa
import time, inspect

class arp(pcs.Packet):
    """ARP"""

    _layout = pcs.Layout()

    def __init__(self, bytes = None, timestamp = None, **kv):
        """initialize an ARP packet"""
        hrd = pcs.Field("hrd", 16, default = 1)
        pro = pcs.Field("pro", 16, default = 0x800)
        hln = pcs.Field("hln", 8, default = 6)
        pln = pcs.Field("pln", 8, default = 4)
        op = pcs.Field("op", 16)
        sha = pcs.StringField("sha", 48)
        spa = pcs.Field("spa", 32)
        tha = pcs.StringField("tha", 48)
        tpa = pcs.Field("tpa", 32)

        pcs.Packet.__init__(self, [hrd, pro, hln, pln, op,
                                   sha, spa, tha, tpa], bytes = bytes, **kv)
        self.description = inspect.getdoc(self)
        if timestamp == None:
            self.timestamp = time.time()
        else:
            self.timestamp = timestamp

        self.data = None

    def __str__(self):
        """return a human readable version of an ARP packet"""
        retval = "ARP\n"
        retval += "hrd: "
        retval += "%d\n" % self.hrd
        retval += "pro: "
        retval += "%d\n" % self.pro
        retval += "hln: "
        retval += "%d\n" % self.hln
        retval += "pln: "
        retval += "%d\n" % self.pln
        retval += "op: "
        retval += "%d\n" % self.op

        retval += "sha: "
        if len(self.sha) >= 6:
            for byte in range(0,5):
                retval += "%s:" % hex(ord(self.sha[byte]))[2:4]
            retval += "%s\n" % hex(ord(self.sha[5]))[2:4]

        retval += "spa: "
        retval += "%s\n" % inet_ntop(AF_INET, struct.pack('!L', self.spa))

        retval += "tha: "
        if len(self.tha) >= 6:
            for byte in range(0,5):
                retval += "%s:" % hex(ord(self.tha[byte]))[2:4]
            retval += "%s\n" % hex(ord(self.tha[5]))[2:4]

        retval += "tpa: "
        retval += "%s\n" % inet_ntop(AF_INET, struct.pack('!L', self.tpa))

        return retval

#
# Functions defined for the module.
#
def ether_atob(pretty):
    """Take a pretty version of an ethernet address and convert it to a
    string of bytes.

    The input string MUST be of the form xx:yy:zz:aa:bb:cc and leading
    zero's must be supplied.  Nor error checking is performed.
    """
    addr = ""
    for i in 0, 3, 6, 9, 12, 15:
        addr += "%c" % int(pretty[i:i+2], 16)
        return addr

def ether_btoa(bytes):
    """Take a set of bytes and convert them to a pretty version of
    and Ethernet address.

    The input buffer MUST be at least 6 bytes long and bytes after the
    sixth are ignored.  No error checking is performed.
    """

    pretty = ""
    for i in (range(5)):
        pretty += hex(bytes[i])[2:4] # Strip the 0x from the string
        pretty += ':'

    pretty += hex(bytes[5])[2:4] # Strip the 0x from the string

    return pretty
<commit_msg>Add ARP definitions from FreeBSD net/if_arp.h<commit_after># Copyright (c) 2006, Neville-Neil Consulting
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# Neither the name of Neville-Neil Consulting nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# File: $Id: arp.py,v 1.2 2006/08/01 13:35:58 gnn Exp $
#
# Author: George V. Neville-Neil
#
# Description:  ARP packet class
import pcs
import struct
from socket import AF_INET, inet_ntop, inet_ntoa
import time, inspect

ARPHRD_ETHER = 1	# ethernet hardware format
ARPHRD_IEEE802 = 6	# token-ring hardware format
ARPHRD_ARCNET = 7	# arcnet hardware format
ARPHRD_FRELAY = 15	# frame relay hardware format
ARPHRD_IEEE1394 = 24	# firewire hardware format

ARPOP_REQUEST = 1	# request to resolve address
ARPOP_REPLY = 2		# response to previous request
ARPOP_REVREQUEST = 3	# request protocol address given hardware
ARPOP_REVREPLY = 4	# response giving protocol address
ARPOP_INVREQUEST = 8	# request to identify peer
ARPOP_INVREPLY = 9	# response identifying peer

class arp(pcs.Packet):
    """ARP"""

    _layout = pcs.Layout()

    def __init__(self, bytes = None, timestamp = None, **kv):
        """initialize an ARP packet"""
        hrd = pcs.Field("hrd", 16, default = 1)
        pro = pcs.Field("pro", 16, default = 0x800)
        hln = pcs.Field("hln", 8, default = 6)
        pln = pcs.Field("pln", 8, default = 4)
        op = pcs.Field("op", 16)
        sha = pcs.StringField("sha", 48)
        spa = pcs.Field("spa", 32)
        tha = pcs.StringField("tha", 48)
        tpa = pcs.Field("tpa", 32)

        pcs.Packet.__init__(self, [hrd, pro, hln, pln, op,
                                   sha, spa, tha, tpa], bytes = bytes, **kv)
        self.description = inspect.getdoc(self)
        if timestamp == None:
            self.timestamp = time.time()
        else:
            self.timestamp = timestamp

        self.data = None

    def __str__(self):
        """return a human readable version of an ARP packet"""
        retval = "ARP\n"
        retval += "hrd: "
        retval += "%d\n" % self.hrd
        retval += "pro: "
        retval += "%d\n" % self.pro
        retval += "hln: "
        retval += "%d\n" % self.hln
        retval += "pln: "
        retval += "%d\n" % self.pln
        retval += "op: "
        retval += "%d\n" % self.op

        retval += "sha: "
        if len(self.sha) >= 6:
            for byte in range(0,5):
                retval += "%s:" % hex(ord(self.sha[byte]))[2:4]
            retval += "%s\n" % hex(ord(self.sha[5]))[2:4]

        retval += "spa: "
        retval += "%s\n" % inet_ntop(AF_INET, struct.pack('!L', self.spa))

        retval += "tha: "
        if len(self.tha) >= 6:
            for byte in range(0,5):
                retval += "%s:" % hex(ord(self.tha[byte]))[2:4]
            retval += "%s\n" % hex(ord(self.tha[5]))[2:4]

        retval += "tpa: "
        retval += "%s\n" % inet_ntop(AF_INET, struct.pack('!L', self.tpa))

        return retval

#
# Functions defined for the module.
#
def ether_atob(pretty):
    """Take a pretty version of an ethernet address and convert it to a
    string of bytes.

    The input string MUST be of the form xx:yy:zz:aa:bb:cc and leading
    zero's must be supplied.  Nor error checking is performed.
    """
    addr = ""
    for i in 0, 3, 6, 9, 12, 15:
        addr += "%c" % int(pretty[i:i+2], 16)
        return addr

def ether_btoa(bytes):
    """Take a set of bytes and convert them to a pretty version of
    and Ethernet address.

    The input buffer MUST be at least 6 bytes long and bytes after the
    sixth are ignored.  No error checking is performed.
    """

    pretty = ""
    for i in (range(5)):
        pretty += hex(bytes[i])[2:4] # Strip the 0x from the string
        pretty += ':'

    pretty += hex(bytes[5])[2:4] # Strip the 0x from the string

    return pretty
<|endoftext|>


<commit_before>import numpy as np
import pandas as pd
from six import iteritems
from six.moves import zip

from zipline.utils.numpy_utils import np_NaT

def next_date_frame(dates, announcement_dates):
    """
    Make a DataFrame representing simulated next earnings dates.

    Parameters
    ----------
    dates : pd.DatetimeIndex.
        The index of the returned DataFrame.
    announcement_dates : dict[int -> pd.Series]
        Dict mapping sids to an index of dates on which earnings were announced
        for that sid.

    Returns
    -------
    next_earnings: pd.DataFrame
        A DataFrame representing, for each (label, date) pair, the first entry
        in `earnings_calendars[label]` on or after `date`.  Entries falling
        after the last date in a calendar will have `np_NaT` as the result in
        the output.

    See Also
    --------
    previous_earnings_date_frame
    """
    cols = {
        equity: np.full_like(dates, np_NaT) for equity in announcement_dates
    }
    raw_dates = dates.values
    for equity, earnings_dates in iteritems(announcement_dates):
        data = cols[equity]
        if not earnings_dates.index.is_monotonic_increasing:
            earnings_dates = earnings_dates.sort_index()

        # Iterate over the raw Series values, since we're comparing against
        # numpy arrays anyway.
        iterkv = zip(earnings_dates.index.values, earnings_dates.values)
        for timestamp, announce_date in iterkv:
            date_mask = (timestamp <= raw_dates) & (raw_dates <= announce_date)
            value_mask = (announce_date <= data) | (data == np_NaT)
            data[date_mask & value_mask] = announce_date

    return pd.DataFrame(index=dates, data=cols)

def previous_date_frame(dates, announcement_dates):
    """
    Make a DataFrame representing simulated next earnings dates.

    Parameters
    ----------
    dates : DatetimeIndex.
        The index of the returned DataFrame.
    announcement_dates : dict[int -> DatetimeIndex]
        Dict mapping sids to an index of dates on which earnings were announced
        for that sid.

    Returns
    -------
    prev_earnings: pd.DataFrame
        A DataFrame representing, for (label, date) pair, the first entry in
        `announcement_dates[label]` strictly before `date`.  Entries falling
        before the first date in a calendar will have `NaT` as the result in
        the output.

    See Also
    --------
    next_earnings_date_frame
    """
    sids = list(announcement_dates)
    out = np.full((len(dates), len(sids)), np_NaT, dtype='datetime64[ns]')
    dn = dates[-1].asm8
    for col_idx, sid in enumerate(sids):
        # announcement_dates[sid] is Series mapping knowledge_date to actual
        # announcement date.  We don't care about the knowledge date for
        # computing previous earnings.
        values = announcement_dates[sid].values
        values = values[values <= dn]
        out[dates.searchsorted(values), col_idx] = values

    frame = pd.DataFrame(out, index=dates, columns=sids)
    frame.ffill(inplace=True)
    return frame
<commit_msg>MAINT: rename variables in (next|previous)_date_frame to be more general<commit_after>import numpy as np
import pandas as pd
from six import iteritems
from six.moves import zip

from zipline.utils.numpy_utils import np_NaT

def next_date_frame(dates, events_by_sid):
    """
    Make a DataFrame representing the simulated next known date for an event.

    Parameters
    ----------
    dates : pd.DatetimeIndex.
        The index of the returned DataFrame.
    events_by_sid : dict[int -> pd.Series]
        Dict mapping sids to a series of dates. Each k:v pair of the series
        represents the date we learned of the event mapping to the date the
        event will occur.
    Returns
    -------
    next_events: pd.DataFrame
        A DataFrame where each column is a security from `events_by_sid` where
        the values are the dates of the next known event with the knowledge we
        had on the date of the index. Entries falling after the last date will
        have `NaT` as the result in the output.

    See Also
    --------
    previous_date_frame
    """
    cols = {
        equity: np.full_like(dates, np_NaT) for equity in events_by_sid
    }
    raw_dates = dates.values
    for equity, event_dates in iteritems(events_by_sid):
        data = cols[equity]
        if not event_dates.index.is_monotonic_increasing:
            event_dates = event_dates.sort_index()

        # Iterate over the raw Series values, since we're comparing against
        # numpy arrays anyway.
        iterkv = zip(event_dates.index.values, event_dates.values)
        for knowledge_date, event_date in iterkv:
            date_mask = (
                (knowledge_date <= raw_dates) &
                (raw_dates <= event_date)
            )
            value_mask = (event_date <= data) | (data == np_NaT)
            data[date_mask & value_mask] = event_date

    return pd.DataFrame(index=dates, data=cols)

def previous_date_frame(date_index, events_by_sid):
    """
    Make a DataFrame representing simulated next earnings date_index.

    Parameters
    ----------
    date_index : DatetimeIndex.
        The index of the returned DataFrame.
    events_by_sid : dict[int -> DatetimeIndex]
        Dict mapping sids to a series of dates. Each k:v pair of the series
        represents the date we learned of the event mapping to the date the
        event will occur.

    Returns
    -------
    previous_events: pd.DataFrame
        A DataFrame where each column is a security from `events_by_sid` where
        the values are the dates of the previous event that occured on the date
        of the index. Entries falling before the first date will have `NaT` as
        the result in the output.

    See Also
    --------
    next_date_frame
    """
    sids = list(events_by_sid)
    out = np.full((len(date_index), len(sids)), np_NaT, dtype='datetime64[ns]')
    dn = date_index[-1].asm8
    for col_idx, sid in enumerate(sids):
        # events_by_sid[sid] is Series mapping knowledge_date to actual
        # event_date.  We don't care about the knowledge date for
        # computing previous earnings.
        values = events_by_sid[sid].values
        values = values[values <= dn]
        out[date_index.searchsorted(values), col_idx] = values

    frame = pd.DataFrame(out, index=date_index, columns=sids)
    frame.ffill(inplace=True)
    return frame
<|endoftext|>


<commit_before>"""General purpose event handling routines.

Public domain code, originally from Marcus von Appen:
https://bitbucket.org/marcusva/python-utils/overview

"""

__all__ = ["EventHandler"]

class EventHandler(object):
    """A simple event handling class, which manages callbacks to be
    executed.

    Attributes:
        callbacks: a list of callbacks that have registered to this event
        sender: the sender (captured by the constructor)

    \section using Subscribing to events
    This class uses C# syntax for subscribing/unsubscribing to events. You may either declare a plain function as a handler, or you can use a lambda.

    Regardless, the first argument with which your method will be called is the "sender" of the event, which is always the class the %EventHandler is instantiated inside (it's *not* the %EventHandler object itself).

    The second argument is the event. Unlike our strongly-typed EventArgs classes in C#, Java, and C++, this event class is more Pythonic: raw event values are published as the second argument of this event handler, removing the need to unpack the event.

    Because this is not strongly typed, be sure to read the documentation so you know what type to expect.

    Here's an example of subscribing to the Pin.analog_voltage_changed event using a lambda:

        >>> board = find_boards()[0]
        >>> board.connect()
        >>> pin = board.pins[0]
        >>> pin.mode = PinMode.AnalogInput
        >>> pin.analog_voltage_changed += lambda sender, value: print(value)

    Though you could just as easily pass the name of a function in place of the `lambda` in-line statement. Imagine having a ``, then passing it to the event:

        >>> def my_event_handler(sender, value):
        >>>     print(value)
        >>>
        >>> pin.analog_voltage_changed += my_event_handler

    \section publishing Publishing events
    You can use this module in your own classes to publish events. Simply initialize a variable as an    EventHandler:

        >>> my_event = EventHandler(self)

    Then call the event just like a function:

        >>> my_event(0.5)

    """

    def __init__(self, sender):
        """Construct a new %EventHandler"""
        self.callbacks = []
        self.sender = sender

    def __call__(self, *args):
        """Executes all callbacks.

        Executes all connected callbacks in the order of addition,
        passing the sender of the %EventHandler as first argument and the
        optional args as second, third, ... argument to them.
        """
        return [callback(self.sender, *args) for callback in self.callbacks]

    def __iadd__(self, callback):
        """Adds a callback to the %EventHandler."""
        self.add(callback)
        return self

    def __isub__(self, callback):
        """Removes a callback from the %EventHandler."""
        self.remove(callback)
        return self

    def __len__(self):
        """Gets the amount of callbacks connected to the %EventHandler."""
        return len(self.callbacks)

    def __getitem__(self, index):
        """Gets the callback at the specified index."""
        return self.callbacks[index]

    def __setitem__(self, index, value):
        """Sets the callback at the specified index."""
        self.callbacks[index] = value

    def __delitem__(self, index):
        """Deletes the callback at the specified index."""
        del self.callbacks[index]

    def add(self, callback):
        """Adds a callback to the %EventHandler."""
        if not callable(callback):
            raise TypeError("callback mus be callable")
        self.callbacks.append(callback)

    def remove(self, callback):
        """Removes a callback from the %EventHandler."""
        self.callbacks.remove(callback)
<commit_msg>Extend event handler<commit_after>"""General purpose event handling routines.

Public domain code, originally from Marcus von Appen:
https://bitbucket.org/marcusva/python-utils/overview

"""

__all__ = ["EventHandler"]

class EventHandler(object):
    """An event handling class, which manages callbacks to be executed.

    \section using Subscribing to events
    This class allows you to publish events that others can subscribe to. Multiple subscribers are supported.

    This class uses += and -= syntax for subscribing/unsubscribing to events.

    You may either declare a plain function as a handler, or you can use a lambda. Regardless, the first argument
    with which your method will be called is the "sender" of the event, which is always the instance of the class the
    %EventHandler is in (it's *not* the %EventHandler object itself).

    The raw event value(s) are published starting at the second positional argument of this event handler.

    Here's an example of subscribing to the Pin.analog_voltage_changed event using a lambda:

        >>> board = find_boards()[0]
        >>> board.connect()
        >>> pin = board.pins[0]
        >>> pin.mode = PinMode.AnalogInput
        >>> pin.analog_voltage_changed += lambda sender, value: print(value)

    You can also use plain functions as event handlers:

        >>> def my_event_handler(sender, value):
        >>>     print(value)
        >>>
        >>> pin.analog_voltage_changed += my_event_handler

    \section publishing Publishing events
    You can use this module in your own classes to publish events. Simply initialize a variable as an EventHandler:

        >>> my_event = EventHandler(self)

    Then call the event just like a function:

        >>> my_event(0.5)

    Subscribers to your event will be called in order they subscribed.
    """

    def __init__(self, sender):
        """Construct a new %EventHandler"""
        self._callbacks = []
        self._sender = sender

    def __call__(self, *args):
        """Executes all callbacks.

        Executes all connected callbacks in the order of addition,
        passing the sender of the %EventHandler as first argument and the
        optional args as second, third, ... argument to them.
        """
        return [callback(self._sender, *args) for callback in self._callbacks]

    def __iadd__(self, callback):
        """Adds a callback to the %EventHandler."""
        self.add(callback)
        return self

    def __isub__(self, callback):
        """Removes a callback from the %EventHandler."""
        self.remove(callback)
        return self

    def __len__(self):
        """Gets the amount of callbacks connected to the %EventHandler."""
        return len(self._callbacks)

    def __getitem__(self, index):
        """Gets the callback at the specified index."""
        return self._callbacks[index]

    def __setitem__(self, index, value):
        """Sets the callback at the specified index."""
        self._callbacks[index] = value

    def __delitem__(self, index):
        """Deletes the callback at the specified index."""
        del self._callbacks[index]

    def add(self, callback):
        """Adds a callback to the %EventHandler."""
        if not callable(callback):
            raise TypeError("callback mus be callable")
        self._callbacks.append(callback)

    def remove(self, callback):
        """Removes a callback from the %EventHandler."""
        self._callbacks.remove(callback)
<|endoftext|>