<commit_before>"""
Runs a pre-caching server against an ORES instance by listening to an RCStream
and submitting requests for scores to the web interface for each revision as it
happens.

If ran by systemd, it uses watchdog to stay alive.

:Usage:
    precached -h | --help
    precached <stream-url> <ores-url> [--config=<path>] [--delay=<secs>]
                                      [--verbose]

:Options:
    -h --help        Prints this documentation
    <stream-url>     URL of an RCStream
    <ores-url>       URL of base ORES webserver
    --delay=<secs>   The delay between when an event is received and when the
                     request is sent to ORES. [default: 0]
    --config=<path>  The path to a yaml config file
                     [default: config]
    --verbose        Print debugging information
"""
import json
import glob
import logging
import os
import sys
import time
from collections import defaultdict
from sseclient import SSEClient as EventSource

import docopt
import requests
import yamlconf

from ..metrics_collectors import MetricsCollector, Null
from .watchdog import notify_socket, watchdog_ping

logger = logging.getLogger(__name__)

AVAILABLE_EVENTS = {'edit', 'nonbot_edit'}
MAX_WORKERS = 50

def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    stream_url = args['<stream-url>']
    ores_url = args['<ores-url>']
    config_paths = os.path.join(args['--config'], "*.yaml")
    config = yamlconf.load(*(open(p) for p in
                             sorted(glob.glob(config_paths))))
    delay = float(args['--delay'])
    verbose = bool(args['--verbose'])
    notify = notify_socket()
    ss_name = config['ores']['scoring_system']
    if 'metrics_collector' in config['scoring_systems'][ss_name]:
        metrics_collector = MetricsCollector.from_config(
            config, config['scoring_systems'][ss_name]['metrics_collector'])
    else:
        metrics_collector = Null()

    run(stream_url, ores_url, metrics_collector, config, delay,
        notify, verbose)

def run(stream_url, ores_url, metrics_collector, config, delay, notify, verbose):

    if verbose:
        log_level = logging.DEBUG
    else:
        log_level = logging.INFO

    logging.basicConfig(
        level=log_level,
        format='%(asctime)s %(levelname)s:%(name)s -- %(message)s'
    )

    logging.getLogger("requests").setLevel(logging.WARNING)
    requests.packages.urllib3.disable_warnings()
    # If we're using logging for metrics collection, show it.
    logging.getLogger("ores.metrics_collectors").setLevel(logging.DEBUG)

    if not notify:
        logger.info('Not being ran as a service, watchdog disabled')

    score_on = build_score_on(config)
    RCNamespace = build_RCNamespace(
        stream_url, score_on, ores_url, MAX_WORKERS, metrics_collector, delay,
        notify)

    for event in EventSource(stream_url):
        if event.event == 'message':
            try:
                change = json.loads(event.data)
            except ValueError:
                pass
            else:
                RCNamespace.on_change(change)

def build_score_on(config):
    # Build a mapping of wikis and models from the configuration
    score_on = defaultdict(list)
    ss_name = config['ores']['scoring_system']
    for context in config['scoring_systems'][ss_name]['scoring_contexts']:
        for model in config['scoring_contexts'][context].get('precache', []):
            precached_config = \
                config['scoring_contexts'][context]['precache'][model]

            events = precached_config['on']
            if len(set(events) - AVAILABLE_EVENTS) > 0:
                logger.error("{0} events are not available"
                             .format(set(events) - AVAILABLE_EVENTS))
                sys.exit(1)
            for event in precached_config['on']:
                score_on[(event, context)].append(model)
                logger.debug("Setting up precaching for {0} in {1} on {2}"
                             .format(model, context, event))

    return score_on

def build_RCNamespace(stream_url, score_on, ores_url, max_workers,
                       metrics_collector, delay, notify):

    class _RCNamespace():

        def on_change(self, change):
            if change['type'] in ('new', 'edit'):
                wikidb = change['wiki']
                rev_id = change['revision']['new']
                models = score_on.get(('edit', wikidb), [])
                self._score_revision(wikidb, models, rev_id)

                if not change.get('bot'):
                    models = score_on.get(('nonbot_edit', wikidb), [])
                    self._score_revision(wikidb, models, rev_id)

        def _score_revision(self, context_name, model_names, rev_id):
            if not model_names:
                return
            url = ores_url + "/v2/scores/" + context_name + "/" + \
                  "?models=" + "|".join(model_names) + \
                  "&revids=" + str(rev_id) + \
                  "&precache=true"
            try:
                time.sleep(delay)
                start = time.time()
                response = requests.get(url, timeout=20, verify=True)
                if response.status_code == 200:
                    first_model_doc = \
                        response.json()['scores'][context_name][model_names[0]]
                    if 'error' not in first_model_doc['scores'][str(rev_id)]:
                        metrics_collector.precache_score(
                            context_name, model_names, time.time() - start)
                    else:
                        metrics_collector.precache_scoring_error(
                            context_name, model_names, response.status_code,
                            time.time() - start)
                        logger.warning(
                            "Request to {0} failed with {1} response: \n\t{2}"
                            .format(url, response.status_code,
                                    first_model_doc['scores'][str(rev_id)]))

                else:
                    metrics_collector.precache_scoring_error(
                        context_name, model_names, response.status_code,
                        time.time() - start)
                    logger.warning(
                        "Request to {0} failed with {1} response: \n\t{2}"
                        .format(url, response.status_code, response.content[:500]))
                if notify:
                    watchdog_ping(*notify)
            except requests.exceptions.ReadTimeout:
                metrics_collector.precache_scoring_error(
                    context_name, model_names, 504,
                    time.time() - start)
                logger.warning(
                    "Request to {0} timed out in {1} seconds"
                    .format(url, round(time.time() - start, 3)))
            except Exception as e:
                logger.error(str(type(e)) + ": " + str(e))

    return _RCNamespace
<commit_msg>Simplified precached, use /precache endpoint.<commit_after>"""
Runs a pre-caching server against an ORES instance by listening to an
EventStream and submitting requests for scores to the web interface for each
revision as it happens.

If ran by systemd, it uses watchdog to stay alive.

:Usage:
    precached -h | --help
    precached <stream-url> <ores-url> [--config=<path>] [--delay=<secs>]
                                      [--debug]

:Options:
    -h --help        Prints this documentation
    <stream-url>     URL of an RCStream
    <ores-url>       URL of base ORES webserver
    --delay=<secs>   The delay between when an event is received and when the
                     request is sent to ORES. [default: 0]
    --config=<path>  The path to a yaml config file
                     [default: config]
    --debug          Print debugging information
"""
import glob
import json
import logging
import os
import time
from concurrent.futures import ThreadPoolExecutor

import docopt
import requests
import yamlconf
from sseclient import SSEClient

from ..metrics_collectors import MetricsCollector, Null
from .watchdog import notify_socket, watchdog_ping

logger = logging.getLogger(__name__)

MAX_WORKERS = 50

def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    logging.basicConfig(
        level=logging.INFO if not args['--debug'] else logging.DEBUG,
        format='%(asctime)s %(levelname)s:%(name)s -- %(message)s'
    )
    # Requests is loud.  Be quiet requests.
    logging.getLogger("requests").setLevel(logging.WARNING)
    requests.packages.urllib3.disable_warnings()
    # If we're using logging for metrics collection, show it.
    logging.getLogger("ores.metrics_collectors").setLevel(logging.DEBUG)

    stream_url = args['<stream-url>']
    ores_url = args['<ores-url>']
    config_paths = os.path.join(args['--config'], "*.yaml")
    config = yamlconf.load(*(open(p) for p in
                             sorted(glob.glob(config_paths))))
    delay = float(args['--delay'])
    verbose = bool(args['--verbose'])
    notify = notify_socket()
    if not notify:
        logger.info('Not being ran as a service, watchdog disabled')
    ss_name = config['ores']['scoring_system']
    if 'metrics_collector' in config['scoring_systems'][ss_name]:
        metrics_collector = MetricsCollector.from_config(
            config, config['scoring_systems'][ss_name]['metrics_collector'])
    else:
        metrics_collector = Null()

    run(stream_url, ores_url, metrics_collector, config, delay,
        notify, verbose)

def run(stream_url, ores_url, metrics_collector, config, delay, notify, verbose):

    # What to do in case of a change
    def precache_a_change(change):
        if delay:
            time.sleep(delay)
        if notify:
            watchdog_ping(*notify)
        start = time.time()
        response = requests.post(
            ores_url + "/v3/precache/", data=json.dumps(change))

        if response.status == 200:
            logger.info("Scored {0} in {1} seconds."
                        .format(json.dumps(change), round(time.time() - start, 3)))
        elif response.status == 204:
            logger.debug("Nothing to do for {0}".format(json.dumps(change)))
        else:
            logger.error("Scoring {0} and got an error in response:\n{1}"
                         .format(json.dumps(change), response.content))

    # Execute changes!
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        for event in SSEClient(stream_url):
            if event.event == 'message':
                try:
                    change = json.loads(event.data)
                except ValueError:
                    pass

                executor.submit(precache_a_change, change)
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Copyright (c) 2012 Mikkel Schubert <MSchubert@snm.ku.dk>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
import copy

import paleomix.common.fileutils as fileutils
import paleomix.common.text as text

def _strand_type(value):
    if value not in ("+", "-"):
        raise ValueError("Strand must be '+' or '-', not %r" % (value,))
    return value

_BED_DEFAULTS = ("", 0, 0, "", 0, "+")
_BED_KEYS = ("contig", "start", "end", "name", "score", "strand")
_BED_TYPES = (str, int, int, str, int, _strand_type)

class BEDRecord(object):
    """Class for parsing and representing a BED records.

    The class has the following properties:
       .name -> str
       .start -> int (0-based)
       .end -> int (1-based)
       .name -> str
       .score -> int
       .strand -> '+' or '-'

    These fields can also be accessed using the square brackets notation, which
    also gives access to any additional values, after the strand column. Fields
    default to 0 or empty string, except the strand which defaults to '+'.
    """

    def __init__(self, line=None, _len=None):
        """Constructs a BED record from a line of text. The length of the
        object matches the number of columns in the input line; in the case
        incompatible values, the ValueError exception is raised.

        The len parameter is unused, and included only for compatibility with
        pysam parser objects, such as 'asBED'. No minimum number of columns are
        required, and it is possible to construct an empty bed record.
        """
        self._fields = []

        if line:
            line = line.rstrip("\r\n").split('\t')
            for column, (value, func) in enumerate(zip(line, _BED_TYPES)):
                try:
                    self._fields.append(func(value))
                except ValueError:
                    raise ValueError("Error parsing column %i in BED record "
                                     "(%r); expected type %s, but found %r."
                                     % (column, "\t".join(line),
                                        func.__name__, value,))

            if len(line) > len(self._fields):
                self._fields.extend(line[len(self._fields):])

    def __copy__(self):
        """Needed for copy.copy to work correctly as expected."""
        record = BEDRecord()
        record._fields = copy.copy(self._fields)
        return record

    def __len__(self):
        """Returns the number of fields in the record; 0 .. N."""
        return len(self._fields)

    def __str__(self):
        """Returns a string suitable for writing to a .bed file."""
        return "\t".join(str(value) for value in self._fields)

    def __repr__(self):
        """Returns a printable representation of the record."""
        fields = []
        for name, value in zip(_BED_KEYS, self._fields):
            fields.append("%s=%r" % (name, value))

        fields.extend(repr(value) for value in self._fields[len(_BED_KEYS):])

        return "BEDRecord(%s)" % (", ".join(fields))

    def __getitem__(self, index):
        return self._fields[index]

    def __setitem__(self, index, value):
        if len(self._fields) <= index:
            defaults = _BED_DEFAULTS[len(self._fields):index + 1]
            self._fields.extend(defaults)
            while len(self._fields) <= index:
                self._fields.append("")

        if index < len(_BED_TYPES):
            if type(_BED_TYPES[index]) is type:
                if not isinstance(value, _BED_TYPES[index]):
                    raise ValueError("Expected %s for BED field %i, got %r"
                                     % (_BED_TYPES[index].__name__,
                                        index + 1, value))
            else:
                value = _BED_TYPES[index](value)

        self._fields[index] = value

    def __cmp__(self, other):
        if not isinstance(other, BEDRecord):
            return cmp(self.__class__, other.__class__)

        return cmp(self._fields, other._fields)

    @classmethod
    def _set_properties(cls):
        for index, name in enumerate(_BED_KEYS):
            setattr(cls, name, cls._new_attr(index))

    @classmethod
    def _new_attr(cls, index):
        """Returns an getter / setter property for the given value."""
        def _get(self):
            return self._fields[index]

        def _set(self, value):
            self[index] = value

        return property(_get, _set)

# Fill out properties for BEDRecord
BEDRecord._set_properties()

def read_bed_file(filename):
    """Parses a (gzip/bzip2 compressed) BED file, and yields
    a sequence of records. Comments and empty lines are skipped."""
    handle = None
    try:
        handle = fileutils.open_ro(filename)
        for record in text.parse_lines(handle, BEDRecord):
            yield record
    finally:
        if handle:
            handle.close()

def sort_bed_by_bamfile(bamfile, regions):
    """Orders a set of BED regions, such that processing matches
    (as far as possible) the layout of the BAM file. This may be
    used to ensure that extraction of regions occurs (close to)
    linearly."""
    if not regions:
        return

    indices = dict(zip(bamfile.references,
                   xrange(len(bamfile.references))))

    def _by_bam_layout(region):
        return (indices[region.contig], region.start, region.end)
    regions.sort(key=_by_bam_layout)
<commit_msg>Fix typo in docstring.<commit_after>#!/usr/bin/python
#
# Copyright (c) 2012 Mikkel Schubert <MSchubert@snm.ku.dk>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
import copy

import paleomix.common.fileutils as fileutils
import paleomix.common.text as text

def _strand_type(value):
    if value not in ("+", "-"):
        raise ValueError("Strand must be '+' or '-', not %r" % (value,))
    return value

_BED_DEFAULTS = ("", 0, 0, "", 0, "+")
_BED_KEYS = ("contig", "start", "end", "name", "score", "strand")
_BED_TYPES = (str, int, int, str, int, _strand_type)

class BEDRecord(object):
    """Class for parsing and representing a BED records.

    The class has the following properties:
       .contig -> str
       .start -> int (0-based)
       .end -> int (1-based)
       .name -> str
       .score -> int
       .strand -> '+' or '-'

    These fields can also be accessed using the square brackets notation, which
    also gives access to any additional values, after the strand column. Fields
    default to 0 or empty string, except the strand which defaults to '+'.
    """

    def __init__(self, line=None, _len=None):
        """Constructs a BED record from a line of text. The length of the
        object matches the number of columns in the input line; in the case
        incompatible values, the ValueError exception is raised.

        The len parameter is unused, and included only for compatibility with
        pysam parser objects, such as 'asBED'. No minimum number of columns are
        required, and it is possible to construct an empty bed record.
        """
        self._fields = []

        if line:
            line = line.rstrip("\r\n").split('\t')
            for column, (value, func) in enumerate(zip(line, _BED_TYPES)):
                try:
                    self._fields.append(func(value))
                except ValueError:
                    raise ValueError("Error parsing column %i in BED record "
                                     "(%r); expected type %s, but found %r."
                                     % (column, "\t".join(line),
                                        func.__name__, value,))

            if len(line) > len(self._fields):
                self._fields.extend(line[len(self._fields):])

    def __copy__(self):
        """Needed for copy.copy to work correctly as expected."""
        record = BEDRecord()
        record._fields = copy.copy(self._fields)
        return record

    def __len__(self):
        """Returns the number of fields in the record; 0 .. N."""
        return len(self._fields)

    def __str__(self):
        """Returns a string suitable for writing to a .bed file."""
        return "\t".join(str(value) for value in self._fields)

    def __repr__(self):
        """Returns a printable representation of the record."""
        fields = []
        for name, value in zip(_BED_KEYS, self._fields):
            fields.append("%s=%r" % (name, value))

        fields.extend(repr(value) for value in self._fields[len(_BED_KEYS):])

        return "BEDRecord(%s)" % (", ".join(fields))

    def __getitem__(self, index):
        return self._fields[index]

    def __setitem__(self, index, value):
        if len(self._fields) <= index:
            defaults = _BED_DEFAULTS[len(self._fields):index + 1]
            self._fields.extend(defaults)
            while len(self._fields) <= index:
                self._fields.append("")

        if index < len(_BED_TYPES):
            if type(_BED_TYPES[index]) is type:
                if not isinstance(value, _BED_TYPES[index]):
                    raise ValueError("Expected %s for BED field %i, got %r"
                                     % (_BED_TYPES[index].__name__,
                                        index + 1, value))
            else:
                value = _BED_TYPES[index](value)

        self._fields[index] = value

    def __cmp__(self, other):
        if not isinstance(other, BEDRecord):
            return cmp(self.__class__, other.__class__)

        return cmp(self._fields, other._fields)

    @classmethod
    def _set_properties(cls):
        for index, name in enumerate(_BED_KEYS):
            setattr(cls, name, cls._new_attr(index))

    @classmethod
    def _new_attr(cls, index):
        """Returns an getter / setter property for the given value."""
        def _get(self):
            return self._fields[index]

        def _set(self, value):
            self[index] = value

        return property(_get, _set)

# Fill out properties for BEDRecord
BEDRecord._set_properties()

def read_bed_file(filename):
    """Parses a (gzip/bzip2 compressed) BED file, and yields
    a sequence of records. Comments and empty lines are skipped."""
    handle = None
    try:
        handle = fileutils.open_ro(filename)
        for record in text.parse_lines(handle, BEDRecord):
            yield record
    finally:
        if handle:
            handle.close()

def sort_bed_by_bamfile(bamfile, regions):
    """Orders a set of BED regions, such that processing matches
    (as far as possible) the layout of the BAM file. This may be
    used to ensure that extraction of regions occurs (close to)
    linearly."""
    if not regions:
        return

    indices = dict(zip(bamfile.references,
                   xrange(len(bamfile.references))))

    def _by_bam_layout(region):
        return (indices[region.contig], region.start, region.end)
    regions.sort(key=_by_bam_layout)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#    Author: Nicolas Bessi. Copyright Camptocamp SA
#    Copyright (C)
#       2014:       Agile Business Group (<http://www.agilebg.com>)
#       2015:       Grupo ESOC <www.grupoesoc.es>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

from openerp import api, fields, models
from . import exceptions

class ResPartner(models.Model):
    """Adds last name and first name; name becomes a stored function field."""
    _inherit = 'res.partner'

    firstname = fields.Char("First name")
    lastname = fields.Char("Last name")
    name = fields.Char(
        compute="_name_compute",
        inverse="_name_inverse",
        required=False,
        store=True)

    @api.one
    @api.depends("firstname", "lastname")
    def _name_compute(self):
        """Write the 'name' field according to splitted data."""
        self.name = " ".join((p for p in (self.lastname,
                                          self.firstname) if p))

    @api.one
    def _name_inverse(self):
        """Try to reverse the effect of _compute_name_custom.

        - If the partner is a company, save it in the first name.
        - Otherwise, make a guess.
        """
        # Remove unneeded whitespace
        clean = " ".join(self.name.split(None))

        # Clean name avoiding infinite recursion
        if self.name != clean:
            self.name = clean

        # Save name in the real fields
        else:
            # Company name goes to the lastname
            if self.is_company:
                parts = [clean, False]

            # Guess name splitting
            else:
                parts = clean.split(" ", 1)
                while len(parts) < 2:
                    parts.append(False)

            self.lastname, self.firstname = parts

    @api.one
    @api.constrains("firstname", "lastname")
    def _check_name(self):
        """Ensure at least one name is set."""
        if not (self.firstname or self.lastname):
            raise exceptions.EmptyNames(self)

    @api.model
    def _firstname_install(self):
        """Save names correctly in the database.

        Before installing the module, field ``name`` contains all full names.
        When installing it, this method parses those names and saves them
        correctly into the database. This can be called later too if needed.
        """
        # Find records with empty firstname and lastname
        records = self.search([("firstname", "=", False),
                               ("lastname", "=", False)])

        # Force calculations there
        records._write_name()
<commit_msg>Fix wrong naming.<commit_after># -*- coding: utf-8 -*-

#    Author: Nicolas Bessi. Copyright Camptocamp SA
#    Copyright (C)
#       2014:       Agile Business Group (<http://www.agilebg.com>)
#       2015:       Grupo ESOC <www.grupoesoc.es>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

from openerp import api, fields, models
from . import exceptions

class ResPartner(models.Model):
    """Adds last name and first name; name becomes a stored function field."""
    _inherit = 'res.partner'

    firstname = fields.Char("First name")
    lastname = fields.Char("Last name")
    name = fields.Char(
        compute="_name_compute",
        inverse="_name_inverse",
        required=False,
        store=True)

    @api.one
    @api.depends("firstname", "lastname")
    def _name_compute(self):
        """Write the 'name' field according to splitted data."""
        self.name = " ".join((p for p in (self.lastname,
                                          self.firstname) if p))

    @api.one
    def _name_inverse(self):
        """Try to reverse the effect of :meth:`._check_name`.

        - If the partner is a company, save it in the first name.
        - Otherwise, make a guess.
        """
        # Remove unneeded whitespace
        clean = " ".join(self.name.split(None))

        # Clean name avoiding infinite recursion
        if self.name != clean:
            self.name = clean

        # Save name in the real fields
        else:
            # Company name goes to the lastname
            if self.is_company:
                parts = [clean, False]

            # Guess name splitting
            else:
                parts = clean.split(" ", 1)
                while len(parts) < 2:
                    parts.append(False)

            self.lastname, self.firstname = parts

    @api.one
    @api.constrains("firstname", "lastname")
    def _check_name(self):
        """Ensure at least one name is set."""
        if not (self.firstname or self.lastname):
            raise exceptions.EmptyNames(self)

    @api.model
    def _firstname_install(self):
        """Save names correctly in the database.

        Before installing the module, field ``name`` contains all full names.
        When installing it, this method parses those names and saves them
        correctly into the database. This can be called later too if needed.
        """
        # Find records with empty firstname and lastname
        records = self.search([("firstname", "=", False),
                               ("lastname", "=", False)])

        # Force calculations there
        records._name_inverse()
<|endoftext|>


<commit_before>from datetime import datetime

__author__ = "Tsuyoshi Hombashi"
__copyright__ = "Copyright 2016-{}, {}".format(datetime.now().year, __author__)
__license__ = "MIT License"
__version__ = "2.0.1"
__maintainer__ = __author__
__email__ = "tsuyoshi.hombashi@gmail.com"
<commit_msg>Bump version<commit_after>from datetime import datetime

__author__ = "Tsuyoshi Hombashi"
__copyright__ = "Copyright 2016-{}, {}".format(datetime.now().year, __author__)
__license__ = "MIT License"
__version__ = "2.1.0"
__maintainer__ = __author__
__email__ = "tsuyoshi.hombashi@gmail.com"
<|endoftext|>


<commit_before>import os
import string
import uuid
import json

from PblCommand import PblCommand

SDK_VERSION = "3"

class PblProjectCreator(PblCommand):
    name = 'new-project'
    help = 'Create a new Pebble project'

    def configure_subparser(self, parser):
        parser.add_argument("name", help = "Name of the project you want to create")
        parser.add_argument("--simple", action="store_true", help = "Use a minimal c file")
        parser.add_argument("--javascript", action="store_true", help = "Generate javascript related files")
        parser.add_argument("--worker", action="store_true", help = "Generate background worker related files")

    def run(self, args):
        print "Creating new project {}".format(args.name)

        # User can give a path to a new project dir
        project_path = args.name
        project_name = os.path.split(project_path)[1]
        project_root = os.path.join(os.getcwd(), project_path)

        project_src = os.path.join(project_root, "src")

        # Create directories
        os.makedirs(project_root)
        os.makedirs(os.path.join(project_root, "resources"))
        os.makedirs(project_src)

        # Create main .c file
        with open(os.path.join(project_src, "%s.c" % (project_name)), "w") as f:
            f.write(FILE_SIMPLE_MAIN if args.simple else FILE_DUMMY_MAIN)

        # Add appinfo.json file
        appinfo_dummy = DICT_DUMMY_APPINFO.copy()
        appinfo_dummy['uuid'] = str(uuid.uuid4())
        appinfo_dummy['project_name'] = project_name
        with open(os.path.join(project_root, "appinfo.json"), "w") as f:
            f.write(FILE_DUMMY_APPINFO.substitute(**appinfo_dummy))

        # Add .gitignore file
        with open(os.path.join(project_root, ".gitignore"), "w") as f:
            f.write(FILE_GITIGNORE)

        # Add javascript files if applicable
        if args.javascript:
            project_js_src = os.path.join(project_src, "js")
            os.makedirs(project_js_src)

            with open(os.path.join(project_js_src, "pebble-js-app.js"), "w") as f:
                f.write(FILE_DUMMY_JAVASCRIPT_SRC)

        # Add background worker files if applicable
        if args.worker:
            project_worker_src = os.path.join(project_root, "worker_src")
            os.makedirs(project_worker_src)
            # Add simple source file
            with open(os.path.join(project_worker_src, "%s_worker.c" % (project_name)), "w") as f:
                f.write(FILE_DUMMY_WORKER)

        # Add wscript file
        with open(os.path.join(project_root, "wscript"), "w") as f:
            f.write(FILE_WSCRIPT)

FILE_GITIGNORE = """
# Ignore build generated files
build
"""

FILE_WSCRIPT = """
#
# This file is the default set of rules to compile a Pebble project.
#
# Feel free to customize this to your needs.
#

import os.path

top = '.'
out = 'build'

def options(ctx):
    ctx.load('pebble_sdk')

def configure(ctx):
    ctx.load('pebble_sdk')

def build(ctx):
    ctx.load('pebble_sdk')

    build_worker = os.path.exists('worker_src')
    binaries = []

    for p in ctx.env.TARGET_PLATFORMS:
        ctx.set_env(ctx.all_envs[p])
        app_elf='{}/pebble-app.elf'.format(ctx.env.BUILD_DIR)
        ctx.pbl_program(source=ctx.path.ant_glob('src/**/*.c'),
        target=app_elf)

        if build_worker:
            worker_elf='{}/pebble-worker.elf'.format(ctx.env.BUILD_DIR)
            binaries.append({'platform': p, 'app_elf': app_elf, 'worker_elf': worker_elf})
            ctx.pbl_worker(source=ctx.path.ant_glob('worker_src/**/*.c'),
            target=worker_elf)
        else:
            binaries.append({'platform': p, 'app_elf': app_elf})

    ctx.pbl_bundle(binaries=binaries, js=ctx.path.ant_glob('src/js/**/*.js'))
"""

FILE_WSCRIPT_2 = """
#
# This file is the default set of rules to compile a Pebble project.
#
# Feel free to customize this to your needs.
#
import os.path
top = '.'
out = 'build'
def options(ctx):
    ctx.load('pebble_sdk')
def configure(ctx):
    ctx.load('pebble_sdk')
def build(ctx):
    ctx.load('pebble_sdk')
    ctx.pbl_program(source=ctx.path.ant_glob('src/**/*.c'),
                    target='pebble-app.elf')
    if os.path.exists('worker_src'):
        ctx.pbl_worker(source=ctx.path.ant_glob('worker_src/**/*.c'),
                        target='pebble-worker.elf')
        ctx.pbl_bundle(elf='pebble-app.elf',
                        worker_elf='pebble-worker.elf',
                        js=ctx.path.ant_glob('src/js/**/*.js'))
    else:
        ctx.pbl_bundle(elf='pebble-app.elf',
                        js=ctx.path.ant_glob('src/js/**/*.js'))
"""

FILE_SIMPLE_MAIN = """#include <pebble.h>

int main(void) {
  app_event_loop();
}
"""

FILE_DUMMY_WORKER = """#include <pebble_worker.h>

int main(void) {
  worker_event_loop();
}
"""

FILE_DUMMY_MAIN = """#include <pebble.h>

static Window *window;
static TextLayer *text_layer;

static void select_click_handler(ClickRecognizerRef recognizer, void *context) {
  text_layer_set_text(text_layer, "Select");
}

static void up_click_handler(ClickRecognizerRef recognizer, void *context) {
  text_layer_set_text(text_layer, "Up");
}

static void down_click_handler(ClickRecognizerRef recognizer, void *context) {
  text_layer_set_text(text_layer, "Down");
}

static void click_config_provider(void *context) {
  window_single_click_subscribe(BUTTON_ID_SELECT, select_click_handler);
  window_single_click_subscribe(BUTTON_ID_UP, up_click_handler);
  window_single_click_subscribe(BUTTON_ID_DOWN, down_click_handler);
}

static void window_load(Window *window) {
  Layer *window_layer = window_get_root_layer(window);
  GRect bounds = layer_get_bounds(window_layer);

  text_layer = text_layer_create((GRect) { .origin = { 0, 72 }, .size = { bounds.size.w, 20 } });
  text_layer_set_text(text_layer, "Press a button");
  text_layer_set_text_alignment(text_layer, GTextAlignmentCenter);
  layer_add_child(window_layer, text_layer_get_layer(text_layer));
}

static void window_unload(Window *window) {
  text_layer_destroy(text_layer);
}

static void init(void) {
  window = window_create();
  window_set_click_config_provider(window, click_config_provider);
  window_set_window_handlers(window, (WindowHandlers) {
    .load = window_load,
    .unload = window_unload,
  });
  const bool animated = true;
  window_stack_push(window, animated);
}

static void deinit(void) {
  window_destroy(window);
}

int main(void) {
  init();

  APP_LOG(APP_LOG_LEVEL_DEBUG, "Done initializing, pushed window: %p", window);

  app_event_loop();
  deinit();
}
"""

DICT_DUMMY_APPINFO = {
    'company_name': 'MakeAwesomeHappen',
    'version_code': 1,
    'version_label': '1.0',
    'target_platform': '["aplite", "basalt"]',
    'sdk_version': SDK_VERSION,
    'is_watchface': 'false',
    'app_keys': """{
    "dummy": 0
  }""",
    'resources_media': '[]'
}

FILE_DUMMY_APPINFO = string.Template("""{
  "uuid": "${uuid}",
  "shortName": "${project_name}",
  "longName": "${project_name}",
  "companyName": "${company_name}",
  "versionCode": ${version_code},
  "versionLabel": "${version_label}",
  "sdkVersion": ${sdk_version},
  "targetPlatform": ${target_platform},
  "watchapp": {
    "watchface": ${is_watchface}
  },
  "appKeys": ${app_keys},
  "resources": {
    "media": ${resources_media}
  }
}
""")

FILE_DUMMY_JAVASCRIPT_SRC = """\
Pebble.addEventListener("ready",
    function(e) {
        console.log("Hello world! - Sent from your javascript application.");
    }
);
"""

class PebbleProjectException(Exception):
    pass

class InvalidProjectException(PebbleProjectException):
    pass

class OutdatedProjectException(PebbleProjectException):
    pass

def check_project_directory():
    """Check to see if the current directory matches what is created by PblProjectCreator.run.

    Raises an InvalidProjectException or an OutdatedProjectException if everything isn't quite right.
    """

    if not os.path.isdir('src'):
        raise InvalidProjectException

    app_info_path = os.path.join(os.getcwd(), "appinfo.json")
    with open(app_info_path, "r") as f:
        app_info_json = json.load(f)

    if os.path.islink('pebble_app.ld') \
            or os.path.exists('resources/src/resource_map.json') \
            or not os.path.exists('wscript') \
            or not 'sdkVersion' in app_info_json.keys() \
            or app_info_json["sdkVersion"] != SDK_VERSION:
        raise OutdatedProjectException

def requires_project_dir(func):
    def wrapper(self, args):
        check_project_directory()
        return func(self, args)
    return wrapper

<commit_msg>Fix new projects<commit_after>import os
import string
import uuid
import json

from PblCommand import PblCommand

SDK_VERSION = "3"

class PblProjectCreator(PblCommand):
    name = 'new-project'
    help = 'Create a new Pebble project'

    def configure_subparser(self, parser):
        parser.add_argument("name", help = "Name of the project you want to create")
        parser.add_argument("--simple", action="store_true", help = "Use a minimal c file")
        parser.add_argument("--javascript", action="store_true", help = "Generate javascript related files")
        parser.add_argument("--worker", action="store_true", help = "Generate background worker related files")

    def run(self, args):
        print "Creating new project {}".format(args.name)

        # User can give a path to a new project dir
        project_path = args.name
        project_name = os.path.split(project_path)[1]
        project_root = os.path.join(os.getcwd(), project_path)

        project_src = os.path.join(project_root, "src")

        # Create directories
        os.makedirs(project_root)
        os.makedirs(os.path.join(project_root, "resources"))
        os.makedirs(project_src)

        # Create main .c file
        with open(os.path.join(project_src, "%s.c" % (project_name)), "w") as f:
            f.write(FILE_SIMPLE_MAIN if args.simple else FILE_DUMMY_MAIN)

        # Add appinfo.json file
        appinfo_dummy = DICT_DUMMY_APPINFO.copy()
        appinfo_dummy['uuid'] = str(uuid.uuid4())
        appinfo_dummy['project_name'] = project_name
        with open(os.path.join(project_root, "appinfo.json"), "w") as f:
            f.write(FILE_DUMMY_APPINFO.substitute(**appinfo_dummy))

        # Add .gitignore file
        with open(os.path.join(project_root, ".gitignore"), "w") as f:
            f.write(FILE_GITIGNORE)

        # Add javascript files if applicable
        if args.javascript:
            project_js_src = os.path.join(project_src, "js")
            os.makedirs(project_js_src)

            with open(os.path.join(project_js_src, "pebble-js-app.js"), "w") as f:
                f.write(FILE_DUMMY_JAVASCRIPT_SRC)

        # Add background worker files if applicable
        if args.worker:
            project_worker_src = os.path.join(project_root, "worker_src")
            os.makedirs(project_worker_src)
            # Add simple source file
            with open(os.path.join(project_worker_src, "%s_worker.c" % (project_name)), "w") as f:
                f.write(FILE_DUMMY_WORKER)

        # Add wscript file
        with open(os.path.join(project_root, "wscript"), "w") as f:
            f.write(FILE_WSCRIPT)

FILE_GITIGNORE = """
# Ignore build generated files
build
"""

FILE_WSCRIPT = """
#
# This file is the default set of rules to compile a Pebble project.
#
# Feel free to customize this to your needs.
#

import os.path

top = '.'
out = 'build'

def options(ctx):
    ctx.load('pebble_sdk')

def configure(ctx):
    ctx.load('pebble_sdk')

def build(ctx):
    ctx.load('pebble_sdk')

    build_worker = os.path.exists('worker_src')
    binaries = []

    for p in ctx.env.target_platforms:
        ctx.set_env(ctx.all_envs[p])
        app_elf='{}/pebble-app.elf'.format(ctx.env.BUILD_DIR)
        ctx.pbl_program(source=ctx.path.ant_glob('src/**/*.c'),
        target=app_elf)

        if build_worker:
            worker_elf='{}/pebble-worker.elf'.format(ctx.env.BUILD_DIR)
            binaries.append({'platform': p, 'app_elf': app_elf, 'worker_elf': worker_elf})
            ctx.pbl_worker(source=ctx.path.ant_glob('worker_src/**/*.c'),
            target=worker_elf)
        else:
            binaries.append({'platform': p, 'app_elf': app_elf})

    ctx.pbl_bundle(binaries=binaries, js=ctx.path.ant_glob('src/js/**/*.js'))
"""

FILE_WSCRIPT_2 = """
#
# This file is the default set of rules to compile a Pebble project.
#
# Feel free to customize this to your needs.
#
import os.path
top = '.'
out = 'build'
def options(ctx):
    ctx.load('pebble_sdk')
def configure(ctx):
    ctx.load('pebble_sdk')
def build(ctx):
    ctx.load('pebble_sdk')
    ctx.pbl_program(source=ctx.path.ant_glob('src/**/*.c'),
                    target='pebble-app.elf')
    if os.path.exists('worker_src'):
        ctx.pbl_worker(source=ctx.path.ant_glob('worker_src/**/*.c'),
                        target='pebble-worker.elf')
        ctx.pbl_bundle(elf='pebble-app.elf',
                        worker_elf='pebble-worker.elf',
                        js=ctx.path.ant_glob('src/js/**/*.js'))
    else:
        ctx.pbl_bundle(elf='pebble-app.elf',
                        js=ctx.path.ant_glob('src/js/**/*.js'))
"""

FILE_SIMPLE_MAIN = """#include <pebble.h>

int main(void) {
  app_event_loop();
}
"""

FILE_DUMMY_WORKER = """#include <pebble_worker.h>

int main(void) {
  worker_event_loop();
}
"""

FILE_DUMMY_MAIN = """#include <pebble.h>

static Window *window;
static TextLayer *text_layer;

static void select_click_handler(ClickRecognizerRef recognizer, void *context) {
  text_layer_set_text(text_layer, "Select");
}

static void up_click_handler(ClickRecognizerRef recognizer, void *context) {
  text_layer_set_text(text_layer, "Up");
}

static void down_click_handler(ClickRecognizerRef recognizer, void *context) {
  text_layer_set_text(text_layer, "Down");
}

static void click_config_provider(void *context) {
  window_single_click_subscribe(BUTTON_ID_SELECT, select_click_handler);
  window_single_click_subscribe(BUTTON_ID_UP, up_click_handler);
  window_single_click_subscribe(BUTTON_ID_DOWN, down_click_handler);
}

static void window_load(Window *window) {
  Layer *window_layer = window_get_root_layer(window);
  GRect bounds = layer_get_bounds(window_layer);

  text_layer = text_layer_create((GRect) { .origin = { 0, 72 }, .size = { bounds.size.w, 20 } });
  text_layer_set_text(text_layer, "Press a button");
  text_layer_set_text_alignment(text_layer, GTextAlignmentCenter);
  layer_add_child(window_layer, text_layer_get_layer(text_layer));
}

static void window_unload(Window *window) {
  text_layer_destroy(text_layer);
}

static void init(void) {
  window = window_create();
  window_set_click_config_provider(window, click_config_provider);
  window_set_window_handlers(window, (WindowHandlers) {
    .load = window_load,
    .unload = window_unload,
  });
  const bool animated = true;
  window_stack_push(window, animated);
}

static void deinit(void) {
  window_destroy(window);
}

int main(void) {
  init();

  APP_LOG(APP_LOG_LEVEL_DEBUG, "Done initializing, pushed window: %p", window);

  app_event_loop();
  deinit();
}
"""

DICT_DUMMY_APPINFO = {
    'company_name': 'MakeAwesomeHappen',
    'version_code': 1,
    'version_label': '1.0',
    'target_platform': '["aplite", "basalt"]',
    'sdk_version': SDK_VERSION,
    'is_watchface': 'false',
    'app_keys': """{
    "dummy": 0
  }""",
    'resources_media': '[]'
}

FILE_DUMMY_APPINFO = string.Template("""{
  "uuid": "${uuid}",
  "shortName": "${project_name}",
  "longName": "${project_name}",
  "companyName": "${company_name}",
  "versionCode": ${version_code},
  "versionLabel": "${version_label}",
  "sdkVersion": "${sdk_version}",
  "targetPlatform": ${target_platform},
  "watchapp": {
    "watchface": ${is_watchface}
  },
  "appKeys": ${app_keys},
  "resources": {
    "media": ${resources_media}
  }
}
""")

FILE_DUMMY_JAVASCRIPT_SRC = """\
Pebble.addEventListener("ready",
    function(e) {
        console.log("Hello world! - Sent from your javascript application.");
    }
);
"""

class PebbleProjectException(Exception):
    pass

class InvalidProjectException(PebbleProjectException):
    pass

class OutdatedProjectException(PebbleProjectException):
    pass

def check_project_directory():
    """Check to see if the current directory matches what is created by PblProjectCreator.run.

    Raises an InvalidProjectException or an OutdatedProjectException if everything isn't quite right.
    """

    if not os.path.isdir('src'):
        raise InvalidProjectException

    app_info_path = os.path.join(os.getcwd(), "appinfo.json")
    with open(app_info_path, "r") as f:
        app_info_json = json.load(f)

    if os.path.islink('pebble_app.ld') \
            or os.path.exists('resources/src/resource_map.json') \
            or not os.path.exists('wscript') \
            or not 'sdkVersion' in app_info_json.keys() \
            or app_info_json["sdkVersion"] != SDK_VERSION:
        raise OutdatedProjectException

def requires_project_dir(func):
    def wrapper(self, args):
        check_project_directory()
        return func(self, args)
    return wrapper

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import, division, print_function, unicode_literals
import numpy as np
import gdspy
import uuid
import picwriter.toolkit as tk

class DBR(gdspy.Cell):
    """ Standard Distributed Bragg Reflector Cell class (subclass of gdspy.Cell).  Tapers the input waveguide to a periodic waveguide structure with varying width (1-D photonic crystal).

        Args:
           * **wgt** (WaveguideTemplate):  WaveguideTemplate object
           * **length** (float): Length of the DBR region.
           * **period** (float): Period of the repeated unit.
           * **dc** (float): Duty cycle of the repeated unit (must be a float between 0 and 1.0).
           * **w_phc** (float): Width of the thin section of the waveguide.  w_phc = 0 corresponds to disconnected periodic blocks.

        Keyword Args:
           * **taper_length** (float): Length of the taper between the input/output waveguide and the DBR region.  Defaults to 20.0.
           * **port** (tuple): Cartesian coordinate of the input port.  Defaults to (0,0).
           * **direction** (string): Direction that the DBR will point *towards*, must be of type `'NORTH'`, `'WEST'`, `'SOUTH'`, `'EAST'`.  Defaults to `'EAST'`.

        Members:
           * **portlist** (dict): Dictionary with the relevant port information

        Portlist format:
           * portlist['input'] = {'port': (x1,y1), 'direction': 'dir1'}
           * portlist['output'] = {'port': (x2, y2), 'direction': 'dir2'}

        Where in the above (x1,y1) is the same as the 'port' input, (x2, y2) is the end of the DBR, and 'dir1', 'dir2' are of type `'NORTH'`, `'WEST'`, `'SOUTH'`, `'EAST'`.

    """
    def __init__(self, wgt, length, period, dc, w_phc, taper_length=20.0, port=(0,0), direction='EAST'):
        gdspy.Cell.__init__(self, "DBR--"+str(uuid.uuid4()))

        self.portlist = {}

        self.port = port
        self.trace=[port, tk.translate_point(port, length+2*taper_length, direction)]
        self.direction = direction
        self.length = length
        self.taper_length = taper_length
        self.period = period
        self.dc = dc
        self.w_phc = w_phc
        self.wgt = wgt
        self.wg_spec = {'layer': wgt.wg_layer, 'datatype': wgt.wg_datatype}
        self.clad_spec = {'layer': wgt.clad_layer, 'datatype': wgt.clad_datatype}

        self.type_check_trace()
        self.build_cell()
        self.build_ports()

    def type_check_trace(self):
        trace = []
        """ Round each trace value to the nearest 1e-6 -- prevents
        some typechecking errors
        """
        for t in self.trace:
            trace.append((round(t[0], 6), round(t[1], 5)))
        self.trace = trace
        """ Make sure all waypoints specify 90degree angles.  This might be
        updated in the future to allow for 45deg, or arbitrary bends.  For now,
        though, rotations are supported via gdspy library
        """
        dx = abs(self.trace[1][0]-self.trace[0][0])
        dy = abs(self.trace[1][1]-self.trace[0][1])
        if dx>=1e-6 and dy>=1e-6:
            raise ValueError("Warning! Both waypoints *must* be adjacent horizontally or vertically.")

        """ Make sure the photonic crystal waveguide width is smaller than the waveguide width """
        if self.w_phc > self.wgt.wg_width:
            raise ValueError("Warning! The w_phc parameter must be smaller than the waveguide template wg_width.")

    def build_cell(self):
        # Sequentially build all the geometric shapes using gdspy path functions
        # for waveguide, then add it to the Cell
        angle = tk.get_angle(self.trace[0], self.trace[1])
        # Add waveguide tapers leading to DBR region
        taper = gdspy.Path(self.wgt.wg_width, self.trace[0])
        taper.segment(self.taper_length, direction=angle, final_width=self.w_phc, **self.wg_spec)
        taper.segment(self.length, **self.wg_spec)
        taper.segment(self.taper_length, final_width=self.wgt.wg_width, **self.wg_spec)
        # Cladding for DBR region
        clad = gdspy.Path(2*self.wgt.clad_width+self.wgt.wg_width, self.trace[0])
        clad.segment(tk.dist(self.trace[0], self.trace[1]), direction=angle, **self.clad_spec)

        self.add(taper)
        self.add(clad)

        """ Now add the periodic PhC components """
        num_blocks = 1 + (2*self.taper_length + self.length)//self.period
        blockx = self.period*self.dc
        startx = self.trace[0][0] + self.taper_length + self.length/2.0 -(num_blocks-1)*self.period/2.0 - blockx/2.0
        y0 = self.trace[0][1]
        block_list = []
        for i in xrange(int(num_blocks)):
            x = startx + i*self.period
            block_list.append(gdspy.Rectangle((x, y0-self.wgt.wg_width/2.0), (x+blockx, y0+self.wgt.wg_width/2.0), **self.wg_spec))

        angle=0
        if self.direction=="NORTH":
            angle=np.pi/2.0
        elif self.direction=="WEST":
            angle=np.pi
        elif self.direction=="SOUTH":
            angle=-np.pi/2.0

        for block in block_list:
            block.rotate(angle, self.trace[0])
            self.add(block)

    def build_ports(self):
        # Portlist format:
        # example: example:  {'port':(x_position, y_position), 'direction': 'NORTH'}
        self.portlist["input"] = {'port':self.trace[0], 'direction':tk.flip_direction(self.direction)}
        self.portlist["output"] = {'port':self.trace[1], 'direction':self.direction}

if __name__ == "__main__":
    from . import *
    top = gdspy.Cell("top")
    wgt = WaveguideTemplate(bend_radius=50, resist='+')

    wg1=Waveguide([(0,0), (100,0)], wgt)
    tk.add(top, wg1)

    dbr1 = DBR(wgt, 10.0, 0.85, 0.5, 0.4, **wg1.portlist["output"])
    tk.add(top, dbr1)

    (x1, y1) = dbr1.portlist["output"]["port"]
    wg2=Waveguide([(x1,y1),
                   (x1+100,y1),
                   (x1+100,y1+100)], wgt)
    tk.add(top, wg2)

    dbr2 = DBR(wgt, 10.0, 0.85, 0.5, 0.6, **wg2.portlist["output"])
    tk.add(top, dbr2)

    gdspy.LayoutViewer()
    gdspy.write_gds('dbr.gds', unit=1.0e-6, precision=1.0e-9)
<commit_msg>add dbr<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import, division, print_function, unicode_literals
import numpy as np
import gdspy
import uuid
import picwriter.toolkit as tk

class DBR(gdspy.Cell):
    """ Standard Distributed Bragg Reflector Cell class (subclass of gdspy.Cell).  Tapers the input waveguide to a periodic waveguide structure with varying width (1-D photonic crystal).

        Args:
           * **wgt** (WaveguideTemplate):  WaveguideTemplate object
           * **length** (float): Length of the DBR region.
           * **period** (float): Period of the repeated unit.
           * **dc** (float): Duty cycle of the repeated unit (must be a float between 0 and 1.0).
           * **w_phc** (float): Width of the thin section of the waveguide.  w_phc = 0 corresponds to disconnected periodic blocks.

        Keyword Args:
           * **taper_length** (float): Length of the taper between the input/output waveguide and the DBR region.  Defaults to 20.0.
           * **port** (tuple): Cartesian coordinate of the input port.  Defaults to (0,0).
           * **direction** (string): Direction that the DBR will point *towards*, must be of type `'NORTH'`, `'WEST'`, `'SOUTH'`, `'EAST'`.  Defaults to `'EAST'`.

        Members:
           * **portlist** (dict): Dictionary with the relevant port information

        Portlist format:
           * portlist['input'] = {'port': (x1,y1), 'direction': 'dir1'}
           * portlist['output'] = {'port': (x2, y2), 'direction': 'dir2'}

        Where in the above (x1,y1) is the same as the 'port' input, (x2, y2) is the end of the DBR, and 'dir1', 'dir2' are of type `'NORTH'`, `'WEST'`, `'SOUTH'`, `'EAST'`.

    """
    def __init__(self, wgt, length, period, dc, w_phc, taper_length=20.0, port=(0,0), direction='EAST'):
        gdspy.Cell.__init__(self, "DBR--"+str(uuid.uuid4()))

        self.portlist = {}

        self.port = port
        self.trace=[port, tk.translate_point(port, length+2*taper_length, direction)]
        self.direction = direction
        self.length = length
        self.taper_length = taper_length
        self.period = period
        self.dc = dc
        self.w_phc = w_phc
        self.wgt = wgt
        self.wg_spec = {'layer': wgt.wg_layer, 'datatype': wgt.wg_datatype}
        self.clad_spec = {'layer': wgt.clad_layer, 'datatype': wgt.clad_datatype}

        self.type_check_trace()
        self.build_cell()
        self.build_ports()

    def type_check_trace(self):
        trace = []
        """ Round each trace value to the nearest 1e-6 -- prevents
        some typechecking errors
        """
        for t in self.trace:
            trace.append((round(t[0], 6), round(t[1], 5)))
        self.trace = trace
        """ Make sure all waypoints specify 90degree angles.  This might be
        updated in the future to allow for 45deg, or arbitrary bends.  For now,
        though, rotations are supported via gdspy library
        """
        dx = abs(self.trace[1][0]-self.trace[0][0])
        dy = abs(self.trace[1][1]-self.trace[0][1])
        if dx>=1e-6 and dy>=1e-6:
            raise ValueError("Warning! Both waypoints *must* be adjacent horizontally or vertically.")

        """ Make sure the photonic crystal waveguide width is smaller than the waveguide width """
        if self.w_phc > self.wgt.wg_width:
            raise ValueError("Warning! The w_phc parameter must be smaller than the waveguide template wg_width.")

    def build_cell(self):
        # Sequentially build all the geometric shapes using gdspy path functions
        # for waveguide, then add it to the Cell
        angle = tk.get_angle(self.trace[0], self.trace[1])
        # Add waveguide tapers leading to DBR region
        taper = gdspy.Path(self.wgt.wg_width, self.trace[0])
        taper.segment(self.taper_length, direction=angle, final_width=self.w_phc, **self.wg_spec)
        taper.segment(self.length, **self.wg_spec)
        taper.segment(self.taper_length, final_width=self.wgt.wg_width, **self.wg_spec)
        # Cladding for DBR region
        clad = gdspy.Path(2*self.wgt.clad_width+self.wgt.wg_width, self.trace[0])
        clad.segment(tk.dist(self.trace[0], self.trace[1]), direction=angle, **self.clad_spec)

        self.add(taper)
        self.add(clad)

        """ Now add the periodic PhC components """
        num_blocks = 1 + (2*self.taper_length + self.length)//self.period
        blockx = self.period*self.dc
        startx = self.trace[0][0] + self.taper_length + self.length/2.0 -(num_blocks-1)*self.period/2.0 - blockx/2.0
        y0 = self.trace[0][1]
        block_list = []
        for i in range(int(num_blocks)):
            x = startx + i*self.period
            block_list.append(gdspy.Rectangle((x, y0-self.wgt.wg_width/2.0), (x+blockx, y0+self.wgt.wg_width/2.0), **self.wg_spec))

        angle=0
        if self.direction=="NORTH":
            angle=np.pi/2.0
        elif self.direction=="WEST":
            angle=np.pi
        elif self.direction=="SOUTH":
            angle=-np.pi/2.0

        for block in block_list:
            block.rotate(angle, self.trace[0])
            self.add(block)

    def build_ports(self):
        # Portlist format:
        # example: example:  {'port':(x_position, y_position), 'direction': 'NORTH'}
        self.portlist["input"] = {'port':self.trace[0], 'direction':tk.flip_direction(self.direction)}
        self.portlist["output"] = {'port':self.trace[1], 'direction':self.direction}

if __name__ == "__main__":
    from . import *
    top = gdspy.Cell("top")
    wgt = WaveguideTemplate(bend_radius=50, resist='+')

    wg1=Waveguide([(0,0), (100,0)], wgt)
    tk.add(top, wg1)

    dbr1 = DBR(wgt, 10.0, 0.85, 0.5, 0.4, **wg1.portlist["output"])
    tk.add(top, dbr1)

    (x1, y1) = dbr1.portlist["output"]["port"]
    wg2=Waveguide([(x1,y1),
                   (x1+100,y1),
                   (x1+100,y1+100)], wgt)
    tk.add(top, wg2)

    dbr2 = DBR(wgt, 10.0, 0.85, 0.5, 0.6, **wg2.portlist["output"])
    tk.add(top, dbr2)

    gdspy.LayoutViewer()
    gdspy.write_gds('dbr.gds', unit=1.0e-6, precision=1.0e-9)
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

print('HELLO from model_tool')

import cPickle
import json
import os
import numpy as np
import pandas as pd

from keras.models import load_model
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.utils import to_categorical
from sklearn import metrics

from keras.layers import Embedding
from keras.layers import Dense, Input, Flatten, Dropout
from keras.layers import Conv1D, MaxPooling1D, Embedding, GlobalMaxPooling1D
from keras.models import Model
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras.optimizers import RMSprop, Adam

DEFAULT_EMBEDDINGS_PATH = '../data/glove.6B/glove.6B.100d.txt'
DEFAULT_MODEL_DIR = '../models'

DEFAULT_HPARAMS = {
    'max_sequence_length': 250,
    'max_num_words': 10000,
    'embedding_dim': 100,
    'embedding_trainable': False,
    'learning_rate': 0.00005,
    'stop_early': True,
    'es_patience': 1, # Only relevant if STOP_EARLY = True
    'es_min_delta': 0, # Only relevant if STOP_EARLY = True
    'batch_size': 128,
    'epochs': 20,
    'dropout_rate': 0.3,
    'cnn_filter_sizes': [128, 128, 128],
    'cnn_kernel_sizes': [5,5,5],
    'cnn_pooling_sizes': [5, 5, 40],
    'verbose': True
}

def compute_auc(y_true, y_pred):
    fpr, tpr, _thresholds = metrics.roc_curve(y_true, y_pred)
    return metrics.auc(fpr, tpr)

class ToxModel():
    def __init__(self,
                 model_name = None,
                 model_dir = DEFAULT_MODEL_DIR,
                 hparams = None):
        self.model_dir = model_dir
        self.model_name = model_name
        self.model = None
        self.tokenizer = None
        self.hparams = {}
        if hparams is None:
            hparams = DEFAULT_HPARAMS
        self.update_hparams(hparams)
        if model_name:
            self.load_model_from_name(model_name)
        self.print_hparams()

    def print_hparams(self):
        print('Hyperparameters')
        print('---------------')
        for k, v in self.hparams.iteritems():
            print('{}: {}'.format(k, v))
        print('')

    def update_hparams(self, new_hparams):
        self.hparams.update(new_hparams)

    def get_model_name(self):
        return self.model_name

    def save_hparams(self, model_name):
        self.hparams['model_name'] = model_name
        with open(os.path.join(self.model_dir,
                '%s_hparams.json' % self.model_name), 'w') as f:
            json.dump(self.hparams, f, sort_keys=True)

    def load_model_from_name(self, model_name):
        self.model = load_model(os.path.join(self.model_dir, '%s_model.h5' % model_name))
        self.tokenizer = cPickle.load(open(os.path.join(self.model_dir,
                                                        '%s_tokenizer.pkl' % model_name),
                                           'rb'))
        with open(os.path.join(self.model_dir,
                '%s_hparams.json' % self.model_name), 'r') as f:
            self.hparams = json.load(f)

    def fit_and_save_tokenizer(self, texts):
        """Fits tokenizer on texts and pickles the tokenizer state."""
        self.tokenizer = Tokenizer(num_words = self.hparams['max_num_words'])
        self.tokenizer.fit_on_texts(texts)
        cPickle.dump(self.tokenizer, open(os.path.join(self.model_dir, '%s_tokenizer.pkl' % self.model_name), 'wb'))

    def prep_text(self, texts):
        """Turns text into into padded sequences.

        The tokenizer must be initialized before calling this method.

        Args:
            texts: Sequence of text strings.

        Returns:
            A tokenized and padded text sequence as a model input.
        """
        text_sequences = self.tokenizer.texts_to_sequences(texts)
        return pad_sequences(text_sequences, maxlen=self.hparams['max_sequence_length'])

    def load_embeddings(self, embedding_path = DEFAULT_EMBEDDINGS_PATH):
        embeddings_index = {}
        with open(embedding_path) as f:
            for line in f:
                values = line.split()
                word = values[0]
                coefs = np.asarray(values[1:], dtype='float32')
                embeddings_index[word] = coefs

        self.embedding_matrix = np.zeros((len(self.tokenizer.word_index) + 1, self.hparams['embedding_dim']))
        num_words_in_embedding = 0
        for word, i in self.tokenizer.word_index.items():
            embedding_vector = embeddings_index.get(word)
            if embedding_vector is not None:
                num_words_in_embedding += 1
                # words not found in embedding index will be all-zeros.
                self.embedding_matrix[i] = embedding_vector

    def train(self, training_data_path, validation_data_path, text_column, label_column, model_name):
        self.model_name = model_name
        self.save_hparams(model_name)

        train_data = pd.read_csv(training_data_path)
        valid_data = pd.read_csv(validation_data_path)

        print('Fitting tokenizer...')
        self.fit_and_save_tokenizer(train_data[text_column])
        print('Tokenizer fitted!')

        print('Preparing data...')
        train_text, train_labels = (self.prep_text(train_data[text_column]),
                                    to_categorical(train_data[label_column]))
        valid_text, valid_labels = (self.prep_text(valid_data[text_column]),
                                    to_categorical(valid_data[label_column]))
        print('Data prepared!')

        print('Loading embeddings...')
        self.load_embeddings()
        print('Embeddings loaded!')

        print('Building model graph...')
        self.build_model()
        print('Training model...')

        save_path = os.path.join(self.model_dir, '%s_model.h5' % self.model_name)
        callbacks = [ModelCheckpoint(save_path, save_best_only=True, verbose=self.hparams['verbose'])]

        if self.hparams['stop_early']:
            callbacks.append(EarlyStopping(min_delta=self.hparams['es_min_delta'],
                monitor='val_loss', patience=self.hparams['es_patience'], verbose=self.hparams['verbose'], mode='auto'))

        self.model.fit(train_text,
                       train_labels,
                       batch_size=self.hparams['batch_size'],
                       epochs=self.hparams['epochs'],
                       validation_data=(valid_text, valid_labels),
                       callbacks=callbacks,
                       verbose=2)
        print('Model trained!')
        print('Best model saved to {}'.format(save_path))
        print('Loading best model from checkpoint...')
        self.model = load_model(save_path)
        print('Model loaded!')

    def build_model(self):
        sequence_input = Input(shape=(self.hparams['max_sequence_length'],), dtype='int32')
        embedding_layer = Embedding(len(self.tokenizer.word_index) + 1,
                                    self.hparams['embedding_dim'],
                                    weights=[self.embedding_matrix],
                                    input_length=self.hparams['max_sequence_length'],
                                    trainable=self.hparams['embedding_trainable'])

        embedded_sequences = embedding_layer(sequence_input)
        x = embedded_sequences
        for filter_size, kernel_size, pool_size in zip(self.hparams['cnn_filter_sizes'], self.hparams['cnn_kernel_sizes'], self.hparams['cnn_pooling_sizes']):
            x = self.build_conv_layer(x, filter_size, kernel_size, pool_size)

        x = Flatten()(x)
        x = Dropout(self.hparams['dropout_rate'])(x)
        # TODO(nthain): Parametrize the number and size of fully connected layers
        x = Dense(128, activation='relu')(x)
        preds = Dense(2, activation='softmax')(x)

        rmsprop = RMSprop(lr = self.hparams['learning_rate'])
        self.model = Model(sequence_input, preds)
        self.model.compile(loss='categorical_crossentropy',
                      optimizer=rmsprop,
                      metrics=['acc'])

    def build_conv_layer(self, input_tensor, filter_size, kernel_size, pool_size):
        output = Conv1D(filter_size, kernel_size, activation='relu', padding='same')(input_tensor)
        if pool_size:
            output = MaxPooling1D(pool_size, padding = 'same')(output)
        else:
            # TODO(nthain): This seems broken. Fix.
            output = GlobalMaxPooling1D()(output)
        return output

    def predict(self, texts):
        """Returns model predictions on texts."""
        data = self.prep_text(texts)
        return self.model.predict(data)[:,1]

    def score_auc(self, texts, labels):
        preds = self.predict(texts)
        return compute_auc(labels, preds)

    def summary():
        return self.model.summary()
<commit_msg>Handle construction w/ partial hparams.<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

print('HELLO from model_tool')

import cPickle
import json
import os
import numpy as np
import pandas as pd

from keras.models import load_model
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.utils import to_categorical
from sklearn import metrics

from keras.layers import Embedding
from keras.layers import Dense, Input, Flatten, Dropout
from keras.layers import Conv1D, MaxPooling1D, Embedding, GlobalMaxPooling1D
from keras.models import Model
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras.optimizers import RMSprop, Adam

DEFAULT_EMBEDDINGS_PATH = '../data/glove.6B/glove.6B.100d.txt'
DEFAULT_MODEL_DIR = '../models'

DEFAULT_HPARAMS = {
    'max_sequence_length': 250,
    'max_num_words': 10000,
    'embedding_dim': 100,
    'embedding_trainable': False,
    'learning_rate': 0.00005,
    'stop_early': True,
    'es_patience': 1, # Only relevant if STOP_EARLY = True
    'es_min_delta': 0, # Only relevant if STOP_EARLY = True
    'batch_size': 128,
    'epochs': 20,
    'dropout_rate': 0.3,
    'cnn_filter_sizes': [128, 128, 128],
    'cnn_kernel_sizes': [5,5,5],
    'cnn_pooling_sizes': [5, 5, 40],
    'verbose': True
}

def compute_auc(y_true, y_pred):
    fpr, tpr, _thresholds = metrics.roc_curve(y_true, y_pred)
    return metrics.auc(fpr, tpr)

class ToxModel():
    def __init__(self,
                 model_name = None,
                 model_dir = DEFAULT_MODEL_DIR,
                 hparams = None):
        self.model_dir = model_dir
        self.model_name = model_name
        self.model = None
        self.tokenizer = None
        self.hparams = DEFAULT_HPARAMS.copy()
        if hparams:
            self.update_hparams(hparams)
        if model_name:
            self.load_model_from_name(model_name)
        self.print_hparams()

    def print_hparams(self):
        print('Hyperparameters')
        print('---------------')
        for k, v in self.hparams.iteritems():
            print('{}: {}'.format(k, v))
        print('')

    def update_hparams(self, new_hparams):
        self.hparams.update(new_hparams)

    def get_model_name(self):
        return self.model_name

    def save_hparams(self, model_name):
        self.hparams['model_name'] = model_name
        with open(os.path.join(self.model_dir,
                '%s_hparams.json' % self.model_name), 'w') as f:
            json.dump(self.hparams, f, sort_keys=True)

    def load_model_from_name(self, model_name):
        self.model = load_model(os.path.join(self.model_dir, '%s_model.h5' % model_name))
        self.tokenizer = cPickle.load(open(os.path.join(self.model_dir,
                                                        '%s_tokenizer.pkl' % model_name),
                                           'rb'))
        with open(os.path.join(self.model_dir,
                '%s_hparams.json' % self.model_name), 'r') as f:
            self.hparams = json.load(f)

    def fit_and_save_tokenizer(self, texts):
        """Fits tokenizer on texts and pickles the tokenizer state."""
        self.tokenizer = Tokenizer(num_words = self.hparams['max_num_words'])
        self.tokenizer.fit_on_texts(texts)
        cPickle.dump(self.tokenizer, open(os.path.join(self.model_dir, '%s_tokenizer.pkl' % self.model_name), 'wb'))

    def prep_text(self, texts):
        """Turns text into into padded sequences.

        The tokenizer must be initialized before calling this method.

        Args:
            texts: Sequence of text strings.

        Returns:
            A tokenized and padded text sequence as a model input.
        """
        text_sequences = self.tokenizer.texts_to_sequences(texts)
        return pad_sequences(text_sequences, maxlen=self.hparams['max_sequence_length'])

    def load_embeddings(self, embedding_path = DEFAULT_EMBEDDINGS_PATH):
        embeddings_index = {}
        with open(embedding_path) as f:
            for line in f:
                values = line.split()
                word = values[0]
                coefs = np.asarray(values[1:], dtype='float32')
                embeddings_index[word] = coefs

        self.embedding_matrix = np.zeros((len(self.tokenizer.word_index) + 1, self.hparams['embedding_dim']))
        num_words_in_embedding = 0
        for word, i in self.tokenizer.word_index.items():
            embedding_vector = embeddings_index.get(word)
            if embedding_vector is not None:
                num_words_in_embedding += 1
                # words not found in embedding index will be all-zeros.
                self.embedding_matrix[i] = embedding_vector

    def train(self, training_data_path, validation_data_path, text_column, label_column, model_name):
        self.model_name = model_name
        self.save_hparams(model_name)

        train_data = pd.read_csv(training_data_path)
        valid_data = pd.read_csv(validation_data_path)

        print('Fitting tokenizer...')
        self.fit_and_save_tokenizer(train_data[text_column])
        print('Tokenizer fitted!')

        print('Preparing data...')
        train_text, train_labels = (self.prep_text(train_data[text_column]),
                                    to_categorical(train_data[label_column]))
        valid_text, valid_labels = (self.prep_text(valid_data[text_column]),
                                    to_categorical(valid_data[label_column]))
        print('Data prepared!')

        print('Loading embeddings...')
        self.load_embeddings()
        print('Embeddings loaded!')

        print('Building model graph...')
        self.build_model()
        print('Training model...')

        save_path = os.path.join(self.model_dir, '%s_model.h5' % self.model_name)
        callbacks = [ModelCheckpoint(save_path, save_best_only=True, verbose=self.hparams['verbose'])]

        if self.hparams['stop_early']:
            callbacks.append(EarlyStopping(min_delta=self.hparams['es_min_delta'],
                monitor='val_loss', patience=self.hparams['es_patience'], verbose=self.hparams['verbose'], mode='auto'))

        self.model.fit(train_text,
                       train_labels,
                       batch_size=self.hparams['batch_size'],
                       epochs=self.hparams['epochs'],
                       validation_data=(valid_text, valid_labels),
                       callbacks=callbacks,
                       verbose=2)
        print('Model trained!')
        print('Best model saved to {}'.format(save_path))
        print('Loading best model from checkpoint...')
        self.model = load_model(save_path)
        print('Model loaded!')

    def build_model(self):
        sequence_input = Input(shape=(self.hparams['max_sequence_length'],), dtype='int32')
        embedding_layer = Embedding(len(self.tokenizer.word_index) + 1,
                                    self.hparams['embedding_dim'],
                                    weights=[self.embedding_matrix],
                                    input_length=self.hparams['max_sequence_length'],
                                    trainable=self.hparams['embedding_trainable'])

        embedded_sequences = embedding_layer(sequence_input)
        x = embedded_sequences
        for filter_size, kernel_size, pool_size in zip(self.hparams['cnn_filter_sizes'], self.hparams['cnn_kernel_sizes'], self.hparams['cnn_pooling_sizes']):
            x = self.build_conv_layer(x, filter_size, kernel_size, pool_size)

        x = Flatten()(x)
        x = Dropout(self.hparams['dropout_rate'])(x)
        # TODO(nthain): Parametrize the number and size of fully connected layers
        x = Dense(128, activation='relu')(x)
        preds = Dense(2, activation='softmax')(x)

        rmsprop = RMSprop(lr = self.hparams['learning_rate'])
        self.model = Model(sequence_input, preds)
        self.model.compile(loss='categorical_crossentropy',
                      optimizer=rmsprop,
                      metrics=['acc'])

    def build_conv_layer(self, input_tensor, filter_size, kernel_size, pool_size):
        output = Conv1D(filter_size, kernel_size, activation='relu', padding='same')(input_tensor)
        if pool_size:
            output = MaxPooling1D(pool_size, padding = 'same')(output)
        else:
            # TODO(nthain): This seems broken. Fix.
            output = GlobalMaxPooling1D()(output)
        return output

    def predict(self, texts):
        """Returns model predictions on texts."""
        data = self.prep_text(texts)
        return self.model.predict(data)[:,1]

    def score_auc(self, texts, labels):
        preds = self.predict(texts)
        return compute_auc(labels, preds)

    def summary():
        return self.model.summary()
<|endoftext|>


<commit_before>"""In this script we reproduce the original data analysis conducted by
Haxby et al. in

"Distributed and Overlapping Representations of Faces and Objects in Ventral Temporal Cortex"

(Science 2001)

"""

### Fetch data using nilearn dataset fetcher ################################
# specify how many subjects we want to load
n_subjects = 2

import os
from nilearn import datasets
data_files = datasets.fetch_haxby(n_subjects=n_subjects, fetch_stimuli=True)
# get root folder from first entry in data files
data_dir = os.path.dirname(os.path.dirname(data_files.anat[0]))

### Take a quick look at stimuli if option is set to True ###################
take_a_look_at_stimuli = False

if take_a_look_at_stimuli:
    from plot_haxby_stimuli import plot_stimuli
    plot_stimuli(data_files.stimuli)
### Do first similarity analysis for all subjects in subjects list ##########
subject_ids = [1]

# Load nilearn NiftiMasker, the practical masking and unmasking tool
from nilearn.input_data import NiftiMasker
import numpy as np

for subject_id in subject_ids:

    # create masker object
    ventral_temporal_mask = NiftiMasker(data_files.mask_vt[subject_id])
                                        # detrend=True,
                                        # standardize=True)
    # initialize it
    ventral_temporal_mask.fit()
    # mask the BOLD data
    vt_timecourses = ventral_temporal_mask.transform(
        data_files.func[subject_id])

    # The data consist of 12 concatenated runs of 121 timepoints each,
    # and we will reshape the data to reflect this. This is important, since
    # detrending and standardizing needs to respect this structure

    vt_timecourses = vt_timecourses.reshape(12, 121,
                                            vt_timecourses.shape[-1])

    # thus reshaped we can detrend and standardize the timecourses
    from scipy.signal import detrend
    vt_timecourses = detrend(vt_timecourses, axis=1)
    vt_timecourses = ((vt_timecourses
                       - vt_timecourses.mean(axis=1)[:, np.newaxis, :])
                       / vt_timecourses.std(axis=1)[:, np.newaxis, :])
    # reshape back
    vt_timecourses = vt_timecourses.reshape(12 * 121, -1)
    # We could take a look at some BOLD time courses
    take_a_look_at_bold = False
    if take_a_look_at_bold:
        import matplotlib.pylab as plt
        plt.figure()
        plt.plot(vt_timecourses[:121])
        plt.xlabel("TRs")
        plt.ylabel("Normalized BOLD")
        plt.title("Some BOLD time courses.")

    # load labels
    labels = np.recfromcsv(data_files.session_target[subject_id],
                           delimiter=" ")
    # Possibly take a quick look at them
    print "Label names in dataset:"
    print np.unique(labels['labels'])
    # This outputs:
    # ['bottle' 'cat' 'chair' 'face' 'house' 'rest'
    # 'scissors' 'scrambledpix' 'shoe']

    # Now try a quick and dirty SVM on the ROI
    from sklearn.svm import SVC
    from sklearn.multiclass import OneVsRestClassifier
    from sklearn.cross_validation import cross_val_score
    classifier = OneVsRestClassifier(SVC(C=1., kernel="linear"))

    scores = cross_val_score(classifier, vt_timecourses, labels['labels'],
                             cv=12, n_jobs=12, verbose=True)

    # mean score around .8, chance level is at around .11111
    # Note that I didn't even remove resting state here.
    print "Linear SVM C=1 on ROI"
    print scores
    print "Mean score: %1.2f" % scores.mean()

    # Now full brain ANOVA + Linear SVM

    # need to extract full brain mask
    from nilearn.masking import compute_epi_mask
    brain_mask = compute_epi_mask(data_files.func[subject_id])
    brain_masker = NiftiMasker(brain_mask)
    all_timecourses = brain_masker.fit_transform(data_files.func[subject_id])
    from sklearn.feature_selection import f_classif, SelectKBest
    from sklearn.pipeline import Pipeline
    feature_selection = SelectKBest(f_classif, 500)
    pipeline = Pipeline([("Feature selection", feature_selection),
                         ("Classifier", classifier)])

    scores_anova_svm = cross_val_score(pipeline, all_timecourses,
                                       labels['labels'], cv=12, n_jobs=12,
                                       verbose=True)
    # at the moment, this scores at around .63
    # probably improvable
    print "ANOVA + Linear SVM C=1 on full brain"
    print scores_anova_svm
    print "Mean score: %1.2f" % scores_anova_svm.mean()
<commit_msg>updated svm scores in comment - yes, linear does better<commit_after>"""In this script we reproduce the original data analysis conducted by
Haxby et al. in

"Distributed and Overlapping Representations of Faces and Objects in Ventral Temporal Cortex"

(Science 2001)

"""

### Fetch data using nilearn dataset fetcher ################################
# specify how many subjects we want to load
n_subjects = 2

import os
from nilearn import datasets
data_files = datasets.fetch_haxby(n_subjects=n_subjects, fetch_stimuli=True)
# get root folder from first entry in data files
data_dir = os.path.dirname(os.path.dirname(data_files.anat[0]))

### Take a quick look at stimuli if option is set to True ###################
take_a_look_at_stimuli = False

if take_a_look_at_stimuli:
    from plot_haxby_stimuli import plot_stimuli
    plot_stimuli(data_files.stimuli)
### Do first similarity analysis for all subjects in subjects list ##########
subject_ids = [1]

# Load nilearn NiftiMasker, the practical masking and unmasking tool
from nilearn.input_data import NiftiMasker
import numpy as np

for subject_id in subject_ids:

    # create masker object
    ventral_temporal_mask = NiftiMasker(data_files.mask_vt[subject_id])
                                        # detrend=True,
                                        # standardize=True)
    # initialize it
    ventral_temporal_mask.fit()
    # mask the BOLD data
    vt_timecourses = ventral_temporal_mask.transform(
        data_files.func[subject_id])

    # The data consist of 12 concatenated runs of 121 timepoints each,
    # and we will reshape the data to reflect this. This is important, since
    # detrending and standardizing needs to respect this structure

    vt_timecourses = vt_timecourses.reshape(12, 121,
                                            vt_timecourses.shape[-1])

    # thus reshaped we can detrend and standardize the timecourses
    from scipy.signal import detrend
    vt_timecourses = detrend(vt_timecourses, axis=1)
    vt_timecourses = ((vt_timecourses
                       - vt_timecourses.mean(axis=1)[:, np.newaxis, :])
                       / vt_timecourses.std(axis=1)[:, np.newaxis, :])
    # reshape back
    vt_timecourses = vt_timecourses.reshape(12 * 121, -1)
    # We could take a look at some BOLD time courses
    take_a_look_at_bold = False
    if take_a_look_at_bold:
        import matplotlib.pylab as plt
        plt.figure()
        plt.plot(vt_timecourses[:121])
        plt.xlabel("TRs")
        plt.ylabel("Normalized BOLD")
        plt.title("Some BOLD time courses.")

    # load labels
    labels = np.recfromcsv(data_files.session_target[subject_id],
                           delimiter=" ")
    # Possibly take a quick look at them
    print "Label names in dataset:"
    print np.unique(labels['labels'])
    # This outputs:
    # ['bottle' 'cat' 'chair' 'face' 'house' 'rest'
    # 'scissors' 'scrambledpix' 'shoe']

    # Now try a quick and dirty SVM on the ROI
    from sklearn.svm import SVC
    from sklearn.multiclass import OneVsRestClassifier
    from sklearn.cross_validation import cross_val_score
    classifier = OneVsRestClassifier(SVC(C=1., kernel="linear"))

    scores = cross_val_score(classifier, vt_timecourses, labels['labels'],
                             cv=12, n_jobs=12, verbose=True)

    # mean score around .86, chance level is at around .11111
    # Note that I didn't even remove resting state here.
    print "Linear SVM C=1 on ROI"
    print scores
    print "Mean score: %1.2f" % scores.mean()

    # Now full brain ANOVA + Linear SVM

    # need to extract full brain mask
    from nilearn.masking import compute_epi_mask
    brain_mask = compute_epi_mask(data_files.func[subject_id])
    brain_masker = NiftiMasker(brain_mask)
    all_timecourses = brain_masker.fit_transform(data_files.func[subject_id])
    from sklearn.feature_selection import f_classif, SelectKBest
    from sklearn.pipeline import Pipeline
    feature_selection = SelectKBest(f_classif, 500)
    pipeline = Pipeline([("Feature selection", feature_selection),
                         ("Classifier", classifier)])

    scores_anova_svm = cross_val_score(pipeline, all_timecourses,
                                       labels['labels'], cv=12, n_jobs=12,
                                       verbose=True)
    # at the moment, this scores at around .87
    # probably improvable
    print "ANOVA + Linear SVM C=1 on full brain"
    print scores_anova_svm
    print "Mean score: %1.2f" % scores_anova_svm.mean()
<|endoftext|>