<commit_before># -*- coding: utf-8 -*-
"""
ccp.client
~~~~~~~~~~~~

This module implements the Changelog API.

:license: MIT, see LICENSE for more details.

"""

import requests
import json
from time import time
from pkg_resources import get_distribution

API_HOST = "localhost"
API_PORT = 5000

SEVERITY = dict(INFO=1, NOTIFICATION=2, WARNING=3, ERROR=4, CRITICAL=5)

class Client(object):

    def __init__(self, host=API_HOST, port=API_PORT):
        self.host = host
        self.port = port
        self.endpoint = "/api/events"

    def send(self, message, severity, category="misc"):
        headers = {
            "User-Agent": "ccp/client v.%s" % get_distribution("ccp").version
        }

        url = self.get_url()
        print url
        headers["Content-Type"] = "application/json"
        data = {
            "criticality": "%d" % SEVERITY[severity],
            "unix_timestamp": "%d" % time(),
            "category": category,
            "description": message
        }
        try:
            response = requests.post(url, headers=headers, data=json.dumps(data))

            if "OK" in response.text:
                return True
            else:
                print "Failed to send message to server: %s" % response.text
        except Exception as e:
            raise("Failed to send message to server: %s" % e)

    def get_url(self):
        port = "" if self.port == 80 else ":%d" % self.port
        protocol = "http://"
        base_full_url = "%s%s%s%s" % (protocol, self.host, port, self.endpoint)
        return base_full_url
<commit_msg>Add support for passing severity directly as an int<commit_after># -*- coding: utf-8 -*-
"""
ccp.client
~~~~~~~~~~~~

This module implements the Changelog API.

:license: MIT, see LICENSE for more details.

"""

import requests
import json
from time import time
from pkg_resources import get_distribution

API_HOST = "localhost"
API_PORT = 5000

SEVERITY = dict(INFO=1, NOTIFICATION=2, WARNING=3, ERROR=4, CRITICAL=5)

class Client(object):

    def __init__(self, host=API_HOST, port=API_PORT):
        self.host = host
        self.port = port
        self.endpoint = "/api/events"

    def deflate_severity(self, severity):
        if isinstance(severity, int):
            return severity
        return SEVERITY[severity]

    def send(self, message, severity, category="misc"):
        headers = {
            "User-Agent": "ccp/client v.%s" % get_distribution("ccp").version
        }

        url = self.get_url()
        print url
        headers["Content-Type"] = "application/json"
        data = {
            "criticality": "%d" % self.deflate_severity(severity),
            "unix_timestamp": "%d" % time(),
            "category": category,
            "description": message
        }
        try:
            response = requests.post(url, headers=headers, data=json.dumps(data))

            if "OK" in response.text:
                return True
            else:
                print "Failed to send message to server: %s" % response.text
        except Exception as e:
            raise("Failed to send message to server: %s" % e)

    def get_url(self):
        port = "" if self.port == 80 else ":%d" % self.port
        protocol = "http://"
        base_full_url = "%s%s%s%s" % (protocol, self.host, port, self.endpoint)
        return base_full_url
<|endoftext|>


<commit_before>import os
import argparse
import sys

from ij import IJ
from loci.plugins import BF
from loci.plugins.in import ImporterOptions
#import WindowManager

def split_top(path):
    """Like `os.path.split`, but splitting from the topmost directory.

    Parameters
    ----------
    path : string
        The path to be split.

    Returns
    -------
    (head, tail) : string tuple
        The topmost directory in path (`head`) and the rest of the path
        (`tail`).

    Examples
    --------
    >>> path = '/path/to/some/files'
    >>> split_top(path)
    ('/path', 'to/some/files')
    """
    split = os.path.split
    tails = []
    head = path
    tail = ''
    while len(head) != 0 and head != '/':
        head, tail = split(head)
        tails.append(tail)
    if head == '/':
        tails.append(head)
    tails.reverse()
    head = tails[0]
    if len(tails[1:]) > 0:
        tail = os.path.join(*tails[1:])
    else:
        tail = ''
    return head, tail

def convert_files(out_base, path, files, error_file=None, ignore_masks=False):
    """Convert cellomics .C01 files to TIFF files in a sibling directory.

    This function is designed to be used with `os.walk`.

    Parameters
    ----------
    out_base : string
        The name of the sibling directory in which to place converted files.
    path : string
        The path to the files to be converted.
    files : list of string
        The filenames of files, including non-.C01 files.
    error_file : string, optional
        A file to which to log "corrupted" images.
    ignore_masks : bool, optional
        Ignore files ending in "o1.C01".

    Returns
    -------
    None : Use for side effects only.

    Examples
    --------
    >>> in_dir = 'cellomics_files'
    >>> out_dir = 'tiff_files'
    >>> files = os.listdir(in_dir)
    >>> files
    ['image1.c01', 'image2.c01']
    >>> convert_files(out_dir, in_dir, files)
    >>> os.listdir(out_dir)
    ['image1.tif', 'image2.tif']
    """
    in_base, in_tail = split_top(path)
    out_path = os.path.join(out_base, in_tail)
    if not os.path.isdir(out_path):
        os.makedirs(out_path)
    if error_file is None:
        ferr = sys.stdout
    else:
        ferr = open(os.path.join(out_base, error_file), 'w')
    files = filter(lambda x: x.endswith('.C01'), files)
    if ignore_masks:
        files = filter(lambda x: not x.endswith('o1.C01'), files)
    files = sorted(files)
    for fn in files:
        fin = os.path.join(path, fn)
        print fin
        fout = os.path.join(out_path, fn)[:-4] + '.tif'
        opts = ImporterOptions()
        opts.setUngroupFiles(True)
        if not os.path.exists(fout):
            opts.setId(fin)
            try:
                imp = BF.openImagePlus(opts)[0]
            except:
                ferr.write(fin + '\n')
                ferr.flush()
            else:
                print "creating", fout
                IJ.saveAs(imp, 'Tiff', fout)
                imp.close()
        else:
            print fout, "exists"

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Convert a bunch of Cellomics .C01 files to TIFFs.')
    parser.add_argument('root_path', help='The path containing .C01 files')
    parser.add_argument('out_path', help='The path to output the TIFFs.')
    parser.add_argument('-E', '--error-file', metavar='FILENAME',
                        help='Log problem filenames to the given filename.')
    parser.add_argument('-m', '--ignore-masks', action='store_true',
                        help='Ignore files ending in "o1.C01".')

    print sys.argv
    args = parser.parse_args()
    paths = os.walk(args.root_path)
    for path, dirs, files in paths:
        convert_files(path.replace(args.root_path, args.out_path), path,
                      files, args.error_file, args.ignore_masks)
<commit_msg>Allow lowercase .c01 extension<commit_after>import os
import argparse
import sys

from ij import IJ
from loci.plugins import BF
from loci.plugins.in import ImporterOptions
#import WindowManager

def split_top(path):
    """Like `os.path.split`, but splitting from the topmost directory.

    Parameters
    ----------
    path : string
        The path to be split.

    Returns
    -------
    (head, tail) : string tuple
        The topmost directory in path (`head`) and the rest of the path
        (`tail`).

    Examples
    --------
    >>> path = '/path/to/some/files'
    >>> split_top(path)
    ('/path', 'to/some/files')
    """
    split = os.path.split
    tails = []
    head = path
    tail = ''
    while len(head) != 0 and head != '/':
        head, tail = split(head)
        tails.append(tail)
    if head == '/':
        tails.append(head)
    tails.reverse()
    head = tails[0]
    if len(tails[1:]) > 0:
        tail = os.path.join(*tails[1:])
    else:
        tail = ''
    return head, tail

def convert_files(out_base, path, files, error_file=None, ignore_masks=False):
    """Convert cellomics .C01 files to TIFF files in a sibling directory.

    This function is designed to be used with `os.walk`.

    Parameters
    ----------
    out_base : string
        The name of the sibling directory in which to place converted files.
    path : string
        The path to the files to be converted.
    files : list of string
        The filenames of files, including non-.C01 files.
    error_file : string, optional
        A file to which to log "corrupted" images.
    ignore_masks : bool, optional
        Ignore files ending in "o1.C01".

    Returns
    -------
    None : Use for side effects only.

    Examples
    --------
    >>> in_dir = 'cellomics_files'
    >>> out_dir = 'tiff_files'
    >>> files = os.listdir(in_dir)
    >>> files
    ['image1.c01', 'image2.c01']
    >>> convert_files(out_dir, in_dir, files)
    >>> os.listdir(out_dir)
    ['image1.tif', 'image2.tif']
    """
    in_base, in_tail = split_top(path)
    out_path = os.path.join(out_base, in_tail)
    if not os.path.isdir(out_path):
        os.makedirs(out_path)
    if error_file is None:
        ferr = sys.stdout
    else:
        ferr = open(os.path.join(out_base, error_file), 'w')
    files = filter(lambda x: x.endswith('.C01') or x.endswith('.c01'), files)
    if ignore_masks:
        files = filter(lambda x: not (x.endswith('o1.C01') or
				      x.endswith('o1.c01')), files)
    files = sorted(files)
    for fn in files:
        fin = os.path.join(path, fn)
        print fin
        fout = os.path.join(out_path, fn)[:-4] + '.tif'
        opts = ImporterOptions()
        opts.setUngroupFiles(True)
        if not os.path.exists(fout):
            opts.setId(fin)
            try:
                imp = BF.openImagePlus(opts)[0]
            except:
                ferr.write(fin + '\n')
                ferr.flush()
            else:
                print "creating", fout
                IJ.saveAs(imp, 'Tiff', fout)
                imp.close()
        else:
            print fout, "exists"

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Convert a bunch of Cellomics .C01 files to TIFFs.')
    parser.add_argument('root_path', help='The path containing .C01 files')
    parser.add_argument('out_path', help='The path to output the TIFFs.')
    parser.add_argument('-E', '--error-file', metavar='FILENAME',
                        help='Log problem filenames to the given filename.')
    parser.add_argument('-m', '--ignore-masks', action='store_true',
                        help='Ignore files ending in "o1.C01".')

    print sys.argv
    args = parser.parse_args()
    paths = os.walk(args.root_path)
    for path, dirs, files in paths:
        convert_files(path.replace(args.root_path, args.out_path), path,
                      files, args.error_file, args.ignore_masks)
<|endoftext|>


<commit_before>"""Generate FontBakery's desired_glyph_data.json file.

The desired_glyph_data.json file contains the 'recommended' countour count
for encoded glyphs. The contour counts are derived from fonts which were
chosen for their quality and unique design decisions for particular glyphs.

Why make this?
Visually QAing thousands of glyphs by hand is tiring. Most glyphs can only
be constructured in a handful of ways. This means a glyph's contour count
will only differ slightly amongst different fonts, e.g a 'g' could either
be 2 or 3 contours, depending on whether its double story or single story.
However, a quotedbl should have 2 contours, unless the font belongs to a
display family.

In the future, additional glyph data can be included. A good addition would
be the 'recommended' anchor counts for each glyph.
"""
from __future__ import absolute_import, print_function, unicode_literals

import json
import os
import sys

from fontbakery.utils import download_file, get_font_glyph_data
from fontTools.ttLib import TTFont

class JsonSetEncoder(json.JSONEncoder):
    """Serialise Set objects for json module."""

    def default(self, obj):
        if isinstance(obj, set):
            return list(obj)
        return json.JSONEncoder.default(self, obj)

def collate_fonts_data(fonts_data):
    """Collate individual fonts data into a single glyph data list."""
    glyphs = {}

    for family in fonts_data:
        for glyph in family:
            if glyph['unicode'] not in glyphs:
                glyphs[glyph['unicode']] = glyph
            else:
                c = glyphs[glyph['unicode']]['contours']
                glyphs[glyph['unicode']]['contours'] = c | glyph['contours']
    return glyphs.values()

def main():
    git_ofl_prefix = 'http://github.com/google/fonts/raw/master/ofl/'
    git_ufl_prefix = 'http://github.com/google/fonts/raw/master/ufl/'
    git_apache_prefix = 'http://github.com/google/fonts/raw/master/apache/'

    fonts_urls = [
        git_ofl_prefix + 'sourceserifpro/SourceSerifPro-Bold.ttf',
        git_ofl_prefix + 'rosarivo/Rosarivo-Regular.ttf',
        git_ofl_prefix + 'raleway/Raleway-BlackItalic.ttf',
        git_ofl_prefix + 'librebaskerville/LibreBaskerville-Bold.ttf',
        git_ofl_prefix + 'worksans/WorkSans-Regular.ttf',
        git_ufl_prefix + 'ubuntu/Ubuntu-BoldItalic.ttf',
        git_ofl_prefix + 'vollkorn/Vollkorn-BlackItalic.ttf',
        git_ofl_prefix + 'breeserif/BreeSerif-Regular.ttf',
        git_ofl_prefix + 'carme/Carme-Regular.ttf',
        git_ofl_prefix + 'creteround/CreteRound-Regular.ttf',
        git_ofl_prefix + 'eczar/Eczar-Bold.ttf',
        git_ofl_prefix + 'faunaone/FaunaOne-Regular.ttf',
        git_ofl_prefix + 'hind/Hind-Light.ttf',
        git_ufl_prefix + 'ubuntumono/UbuntuMono-Bold.ttf',
        git_ofl_prefix + 'belgrano/Belgrano-Regular.ttf',
        git_ofl_prefix + 'trirong/Trirong-Light.ttf',
        git_ofl_prefix + 'mitr/Mitr-Regular.ttf',
        git_ofl_prefix + 'overpass/Overpass-Regular.ttf',
        git_ofl_prefix + 'jura/Jura-Regular.ttf',
        git_ofl_prefix + 'overpass/Overpass-Black.ttf',
        git_ofl_prefix + 'montserrat/Montserrat-Regular.ttf',
        git_ofl_prefix + 'montserrat/Montserrat-Black.ttf',
        git_ofl_prefix + 'montserrat/Montserrat-Thin.ttf',
        git_apache_prefix + 'roboto/Roboto-Regular.ttf',
    ]

    fonts_data = []
    for font_url in fonts_urls:
        print('Downloading and generating glyph data for {}'.format(font_url))
        font_ttf = download_file(font_url)
        font = TTFont(font_ttf)
        fonts_data.append(get_font_glyph_data(font))

    print('Collating font data into glyph data file')
    glyph_data = collate_fonts_data(fonts_data)

    script_path = os.path.dirname(__file__)
    glyph_data_path = os.path.join(script_path, '..', 'desired_glyph_data.json')

    print('Saving to {}'.format(glyph_data_path))
    with open(glyph_data_path, 'w') as glyph_file:
        json.dump(glyph_data, glyph_file, indent=4, cls=JsonSetEncoder)
    print('done')

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>generate_glyphdata.py: Silence JsonSetEncoder.default lint warnings<commit_after>"""Generate FontBakery's desired_glyph_data.json file.

The desired_glyph_data.json file contains the 'recommended' countour count
for encoded glyphs. The contour counts are derived from fonts which were
chosen for their quality and unique design decisions for particular glyphs.

Why make this?
Visually QAing thousands of glyphs by hand is tiring. Most glyphs can only
be constructured in a handful of ways. This means a glyph's contour count
will only differ slightly amongst different fonts, e.g a 'g' could either
be 2 or 3 contours, depending on whether its double story or single story.
However, a quotedbl should have 2 contours, unless the font belongs to a
display family.

In the future, additional glyph data can be included. A good addition would
be the 'recommended' anchor counts for each glyph.
"""
from __future__ import absolute_import, print_function, unicode_literals

import json
import os
import sys

from fontbakery.utils import download_file, get_font_glyph_data
from fontTools.ttLib import TTFont

class JsonSetEncoder(json.JSONEncoder):
    """Serialise Set objects for json module."""

    def default(self, obj):  # pylint: disable=arguments-differ,method-hidden
        if isinstance(obj, set):
            return list(obj)
        return json.JSONEncoder.default(self, obj)

def collate_fonts_data(fonts_data):
    """Collate individual fonts data into a single glyph data list."""
    glyphs = {}

    for family in fonts_data:
        for glyph in family:
            if glyph['unicode'] not in glyphs:
                glyphs[glyph['unicode']] = glyph
            else:
                c = glyphs[glyph['unicode']]['contours']
                glyphs[glyph['unicode']]['contours'] = c | glyph['contours']
    return glyphs.values()

def main():
    git_ofl_prefix = 'http://github.com/google/fonts/raw/master/ofl/'
    git_ufl_prefix = 'http://github.com/google/fonts/raw/master/ufl/'
    git_apache_prefix = 'http://github.com/google/fonts/raw/master/apache/'

    fonts_urls = [
        git_ofl_prefix + 'sourceserifpro/SourceSerifPro-Bold.ttf',
        git_ofl_prefix + 'rosarivo/Rosarivo-Regular.ttf',
        git_ofl_prefix + 'raleway/Raleway-BlackItalic.ttf',
        git_ofl_prefix + 'librebaskerville/LibreBaskerville-Bold.ttf',
        git_ofl_prefix + 'worksans/WorkSans-Regular.ttf',
        git_ufl_prefix + 'ubuntu/Ubuntu-BoldItalic.ttf',
        git_ofl_prefix + 'vollkorn/Vollkorn-BlackItalic.ttf',
        git_ofl_prefix + 'breeserif/BreeSerif-Regular.ttf',
        git_ofl_prefix + 'carme/Carme-Regular.ttf',
        git_ofl_prefix + 'creteround/CreteRound-Regular.ttf',
        git_ofl_prefix + 'eczar/Eczar-Bold.ttf',
        git_ofl_prefix + 'faunaone/FaunaOne-Regular.ttf',
        git_ofl_prefix + 'hind/Hind-Light.ttf',
        git_ufl_prefix + 'ubuntumono/UbuntuMono-Bold.ttf',
        git_ofl_prefix + 'belgrano/Belgrano-Regular.ttf',
        git_ofl_prefix + 'trirong/Trirong-Light.ttf',
        git_ofl_prefix + 'mitr/Mitr-Regular.ttf',
        git_ofl_prefix + 'overpass/Overpass-Regular.ttf',
        git_ofl_prefix + 'jura/Jura-Regular.ttf',
        git_ofl_prefix + 'overpass/Overpass-Black.ttf',
        git_ofl_prefix + 'montserrat/Montserrat-Regular.ttf',
        git_ofl_prefix + 'montserrat/Montserrat-Black.ttf',
        git_ofl_prefix + 'montserrat/Montserrat-Thin.ttf',
        git_apache_prefix + 'roboto/Roboto-Regular.ttf',
    ]

    fonts_data = []
    for font_url in fonts_urls:
        print('Downloading and generating glyph data for {}'.format(font_url))
        font_ttf = download_file(font_url)
        font = TTFont(font_ttf)
        fonts_data.append(get_font_glyph_data(font))

    print('Collating font data into glyph data file')
    glyph_data = collate_fonts_data(fonts_data)

    script_path = os.path.dirname(__file__)
    glyph_data_path = os.path.join(script_path, '..', 'desired_glyph_data.json')

    print('Saving to {}'.format(glyph_data_path))
    with open(glyph_data_path, 'w') as glyph_file:
        json.dump(glyph_data, glyph_file, indent=4, cls=JsonSetEncoder)
    print('done')

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>import unicodedata

from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.db import models
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.shortcuts import get_object_or_404
from django.utils import timezone
from django.utils.text import slugify
from django.utils.translation import ugettext_lazy as _

from cfp.choices import TALK_DURATIONS
from events.models import Event
from tinymce.models import HTMLField
from utils.behaviors import Timestampable

def get_active_cfp():
    return get_object_or_404(CallForPaper, pk=settings.ACTIVE_CFP_ID)

def get_applicant_avatar_path(instance, filename):
    return "uploads/applicant_images/{0}/{1}".format(
            slugify(instance.user.email),
            unicodedata.normalize('NFKD', filename).encode('ascii', 'ignore'))

class CallForPaper(models.Model):
    event = models.ForeignKey(Event)
    title = models.CharField(max_length=1024)
    description = HTMLField()
    announcement = HTMLField(blank=True, null=True)
    begin_date = models.DateField()
    end_date = models.DateField(blank=True, null=True)

    def __unicode__(self):
        return self.title

    def is_active(self):
        today = timezone.now().date()
        return today >= self.begin_date and (not self.end_date or today <= self.end_date)

    @property
    def applications(self):
        return self.paperapplication_set.all()

    @property
    def application_count(self):
        return self.paperapplication_set.count()

    @property
    def duration(self):
        return (self.end_date - self.begin_date) if self.end_date else None

class Applicant(models.Model):
    user = models.OneToOneField(settings.AUTH_USER_MODEL, related_name='applicant')
    about = models.TextField()
    biography = models.TextField()
    speaker_experience = models.TextField(null=True, blank=True)
    image = models.ImageField(max_length=255, upload_to=get_applicant_avatar_path)

    def __unicode__(self):
        return self.user.get_full_name()

    @property
    def full_name(self):
        return self.__unicode__()

    @property
    def email(self):
        return self.user.email

    @property
    def twitter(self):
        return self.user.twitter

    @property
    def github(self):
        return self.user.github

class AudienceSkillLevel(models.Model):
    name = models.CharField(max_length=50)

    def __unicode__(self):
        return self.name

    class Meta:
        ordering = ['pk', ]

class PaperApplication(Timestampable):
    cfp = models.ForeignKey(CallForPaper)
    applicant = models.ForeignKey(Applicant, related_name='applications')
    title = models.CharField(
        max_length=255,
        help_text=_('The title of your talk. Keep it short and catchy <b>foo</b>.'),
        verbose_name=_('Title'))
    about = models.TextField(help_text=_('Describe your talk in 140 characters or less.'),
                             verbose_name=_('What\'s it about'))
    abstract = models.TextField(help_text=_('You may go in more depth here. Up to 10 sentnces, please.'),
                                verbose_name=_('Abstract'))
    skill_level = models.ForeignKey(AudienceSkillLevel, verbose_name=_('Audience level'),
                                    help_text=_('Which skill level is this talk most appropriate for?'))
    duration = models.CharField(
            choices=TALK_DURATIONS,
            max_length=255,
            default=TALK_DURATIONS.MIN_25,
            verbose_name=_('Talk Duration Slot'),
            help_text=_('What talk duration slot would you like?'))

    accomodation_required = models.BooleanField(
            'I require accomodation',
            help_text='For people outside of the Zagreb area, we provide 2 nights in a hotel.',
            default=False)

    extra_info = models.TextField(
            'Extra info',
            help_text='Anything else that you would like to let us know?',
            null=True,
            blank=True)

    exclude = models.BooleanField(default=False)

    class Meta:
        ordering = ['title', ]

    def __unicode__(self):
        return u'{} - {} - {} min (CFP #{})'.format(
                self.title,
                self.applicant,
                self.duration,
                self.cfp_id)

    @property
    def votes_count(self):
        return self.votes.all().count()

    @property
    def next(self):
        return PaperApplication.objects.filter(cfp=self.cfp, id__gt=self.id).order_by('id').first()

    @property
    def prev(self):
        return PaperApplication.objects.filter(cfp=self.cfp, id__lt=self.id).order_by('-id').first()

    @property
    def ordinal(self):
        """The ordinal number of the application within it's CFP"""
        return PaperApplication.objects.filter(cfp=self.cfp, id__lt=self.id).count() + 1

    @property
    def has_talk(self):
        return hasattr(self, "talk")

@receiver(post_save, sender=PaperApplication)
def update_talk_instance(sender, instance, created, **kwargs):
    if not settings.ALLOW_TALK_UPDATES:
        return

    try:
        instance.talk.save()
    except ObjectDoesNotExist:
        pass
<commit_msg>Remove accidentaly added foo<commit_after>import unicodedata

from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.db import models
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.shortcuts import get_object_or_404
from django.utils import timezone
from django.utils.text import slugify
from django.utils.translation import ugettext_lazy as _

from cfp.choices import TALK_DURATIONS
from events.models import Event
from tinymce.models import HTMLField
from utils.behaviors import Timestampable

def get_active_cfp():
    return get_object_or_404(CallForPaper, pk=settings.ACTIVE_CFP_ID)

def get_applicant_avatar_path(instance, filename):
    return "uploads/applicant_images/{0}/{1}".format(
            slugify(instance.user.email),
            unicodedata.normalize('NFKD', filename).encode('ascii', 'ignore'))

class CallForPaper(models.Model):
    event = models.ForeignKey(Event)
    title = models.CharField(max_length=1024)
    description = HTMLField()
    announcement = HTMLField(blank=True, null=True)
    begin_date = models.DateField()
    end_date = models.DateField(blank=True, null=True)

    def __unicode__(self):
        return self.title

    def is_active(self):
        today = timezone.now().date()
        return today >= self.begin_date and (not self.end_date or today <= self.end_date)

    @property
    def applications(self):
        return self.paperapplication_set.all()

    @property
    def application_count(self):
        return self.paperapplication_set.count()

    @property
    def duration(self):
        return (self.end_date - self.begin_date) if self.end_date else None

class Applicant(models.Model):
    user = models.OneToOneField(settings.AUTH_USER_MODEL, related_name='applicant')
    about = models.TextField()
    biography = models.TextField()
    speaker_experience = models.TextField(null=True, blank=True)
    image = models.ImageField(max_length=255, upload_to=get_applicant_avatar_path)

    def __unicode__(self):
        return self.user.get_full_name()

    @property
    def full_name(self):
        return self.__unicode__()

    @property
    def email(self):
        return self.user.email

    @property
    def twitter(self):
        return self.user.twitter

    @property
    def github(self):
        return self.user.github

class AudienceSkillLevel(models.Model):
    name = models.CharField(max_length=50)

    def __unicode__(self):
        return self.name

    class Meta:
        ordering = ['pk', ]

class PaperApplication(Timestampable):
    cfp = models.ForeignKey(CallForPaper)
    applicant = models.ForeignKey(Applicant, related_name='applications')
    title = models.CharField(
        max_length=255,
        help_text=_('The title of your talk. Keep it short and catchy.'),
        verbose_name=_('Title'))
    about = models.TextField(help_text=_('Describe your talk in 140 characters or less.'),
                             verbose_name=_('What\'s it about'))
    abstract = models.TextField(help_text=_('You may go in more depth here. Up to 10 sentnces, please.'),
                                verbose_name=_('Abstract'))
    skill_level = models.ForeignKey(AudienceSkillLevel, verbose_name=_('Audience level'),
                                    help_text=_('Which skill level is this talk most appropriate for?'))
    duration = models.CharField(
            choices=TALK_DURATIONS,
            max_length=255,
            default=TALK_DURATIONS.MIN_25,
            verbose_name=_('Talk Duration Slot'),
            help_text=_('What talk duration slot would you like?'))

    accomodation_required = models.BooleanField(
            'I require accomodation',
            help_text='For people outside of the Zagreb area, we provide 2 nights in a hotel.',
            default=False)

    extra_info = models.TextField(
            'Extra info',
            help_text='Anything else that you would like to let us know?',
            null=True,
            blank=True)

    exclude = models.BooleanField(default=False)

    class Meta:
        ordering = ['title', ]

    def __unicode__(self):
        return u'{} - {} - {} min (CFP #{})'.format(
                self.title,
                self.applicant,
                self.duration,
                self.cfp_id)

    @property
    def votes_count(self):
        return self.votes.all().count()

    @property
    def next(self):
        return PaperApplication.objects.filter(cfp=self.cfp, id__gt=self.id).order_by('id').first()

    @property
    def prev(self):
        return PaperApplication.objects.filter(cfp=self.cfp, id__lt=self.id).order_by('-id').first()

    @property
    def ordinal(self):
        """The ordinal number of the application within it's CFP"""
        return PaperApplication.objects.filter(cfp=self.cfp, id__lt=self.id).count() + 1

    @property
    def has_talk(self):
        return hasattr(self, "talk")

@receiver(post_save, sender=PaperApplication)
def update_talk_instance(sender, instance, created, **kwargs):
    if not settings.ALLOW_TALK_UPDATES:
        return

    try:
        instance.talk.save()
    except ObjectDoesNotExist:
        pass
<|endoftext|>


<commit_before>#!/usr/bin/env python2

"""Calculate correlation for parametric maps vs. Gleason scores."""

from __future__ import division, print_function
import argparse
import math
import numpy as np
import scipy.stats

import dwi.patient
import dwi.util

def parse_args():
    """Parse command-line arguments."""
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument('--verbose', '-v', action='count',
            help='be more verbose')
    p.add_argument('--patients', default='patients.txt',
            help='patients file')
    p.add_argument('--pmapdir', nargs='+', required=True,
            help='input pmap directory')
    p.add_argument('--thresholds', nargs='*', default=[],
            help='classification thresholds (group maximums)')
    p.add_argument('--voxel', default='all',
            help='index of voxel to use, or all, sole, mean, median')
    p.add_argument('--multilesion', action='store_true',
            help='use all lesions, not just first for each')
    p.add_argument('--dropok', action='store_true',
            help='allow dropping of files not found')
    args = p.parse_args()
    return args

def correlation(x, y):
    """Calculate correlation with p-value and confidence interval."""
    assert len(x) == len(y)
    if dwi.util.all_equal(x):
        r = p = lower = upper = np.nan
    else:
        #r, p = scipy.stats.pearsonr(x, y)
        #r, p = scipy.stats.kendalltau(x, y)
        r, p = scipy.stats.spearmanr(x, y)
        n = len(x)
        stderr = 1 / math.sqrt(n-3)
        delta = 1.96 * stderr
        lower = math.tanh(math.atanh(r) - delta)
        upper = math.tanh(math.atanh(r) + delta)
    return dict(r=r, p=p, lower=lower, upper=upper)

# Collect all parameters.
args = parse_args()
X, Y = [], []
Params = []
scores = None
for i, pmapdir in enumerate(args.pmapdir):
    data = dwi.patient.read_pmaps(args.patients, pmapdir, args.thresholds,
            voxel=args.voxel, multiroi=args.multilesion, dropok=args.dropok)
    if scores is None:
        scores, groups, group_sizes = dwi.patient.grouping(data)
    for j, param in enumerate(data[0]['params']):
        x, y = [], []
        for d in data:
            for v in d['pmap']:
                x.append(v[j])
                y.append(d['label'])
        X.append(np.asarray(x))
        Y.append(np.asarray(y))
        Params.append('%i:%s' % (i, param))

# Print info.
if args.verbose > 1:
    d = dict(n=len(X[0]),
            ns=len(scores), s=scores,
            ng=len(groups), g=' '.join(map(str, groups)),
            gs=', '.join(map(str, group_sizes)))
    print('Samples: {n}'.format(**d))
    print('Scores: {ns}: {s}'.format(**d))
    print('Groups: {ng}: {g}'.format(**d))
    print('Group sizes: {gs}'.format(**d))

# Print correlations.
if args.verbose > 1:
    print('# param  r  p  lower  upper')
params_maxlen = max(len(p) for p in Params)
for x, y, param in zip(X, Y, Params):
    d = dict(param=param)
    d.update(correlation(x, y))
    if args.verbose:
        s = '{param:%i}  {r:+.3f}  {p:.3f}  {lower:+.3f}  {upper:+.3f}' % params_maxlen
    else:
        s = '{r:+.3f}'
    print(s.format(**d))
<commit_msg>style<commit_after>#!/usr/bin/env python2

"""Calculate correlation for parametric maps vs. Gleason scores."""

from __future__ import division, print_function
import argparse
import math
import numpy as np
import scipy.stats

import dwi.patient
import dwi.util

def parse_args():
    """Parse command-line arguments."""
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument('--verbose', '-v', action='count',
                   help='be more verbose')
    p.add_argument('--patients', default='patients.txt',
                   help='patients file')
    p.add_argument('--pmapdir', nargs='+', required=True,
                   help='input pmap directory')
    p.add_argument('--thresholds', nargs='*', default=[],
                   help='classification thresholds (group maximums)')
    p.add_argument('--voxel', default='all',
                   help='index of voxel to use, or all, sole, mean, median')
    p.add_argument('--multilesion', action='store_true',
                   help='use all lesions, not just first for each')
    p.add_argument('--dropok', action='store_true',
                   help='allow dropping of files not found')
    return p.parse_args()

def correlation(x, y):
    """Calculate correlation with p-value and confidence interval."""
    assert len(x) == len(y)
    if dwi.util.all_equal(x):
        r = p = lower = upper = np.nan
    else:
        #r, p = scipy.stats.pearsonr(x, y)
        #r, p = scipy.stats.kendalltau(x, y)
        r, p = scipy.stats.spearmanr(x, y)
        n = len(x)
        stderr = 1 / math.sqrt(n-3)
        delta = 1.96 * stderr
        lower = math.tanh(math.atanh(r) - delta)
        upper = math.tanh(math.atanh(r) + delta)
    return dict(r=r, p=p, lower=lower, upper=upper)

def main():
    # Collect all parameters.
    args = parse_args()
    X, Y = [], []
    Params = []
    scores = None
    for i, pmapdir in enumerate(args.pmapdir):
        data = dwi.patient.read_pmaps(args.patients, pmapdir, args.thresholds,
                                      voxel=args.voxel,
                                      multiroi=args.multilesion,
                                      dropok=args.dropok)
        if scores is None:
            scores, groups, group_sizes = dwi.patient.grouping(data)
        for j, param in enumerate(data[0]['params']):
            x, y = [], []
            for d in data:
                for v in d['pmap']:
                    x.append(v[j])
                    y.append(d['label'])
            X.append(np.asarray(x))
            Y.append(np.asarray(y))
            Params.append('%i:%s' % (i, param))

    # Print info.
    if args.verbose > 1:
        d = dict(n=len(X[0]),
                 ns=len(scores), s=scores,
                 ng=len(groups), g=' '.join(str(x) for x in groups),
                 gs=', '.join(str(x) for x in group_sizes))
        print('Samples: {n}'.format(**d))
        print('Scores: {ns}: {s}'.format(**d))
        print('Groups: {ng}: {g}'.format(**d))
        print('Group sizes: {gs}'.format(**d))

    # Print correlations.
    if args.verbose > 1:
        print('# param  r  p  lower  upper')
    params_maxlen = max(len(p) for p in Params)
    for x, y, param in zip(X, Y, Params):
        d = dict(param=param)
        d.update(correlation(x, y))
        if args.verbose:
            s = ('{param:%i}  {r:+.3f}  {p:.3f}  {lower:+.3f}  {upper:+.3f}' %
                 params_maxlen)
        else:
            s = '{r:+.3f}'
        print(s.format(**d))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import pprint
import functools

# This look-up table, initialised upon the first call to `from_json`, maps CGum
# AST types to their representative Python classes, using the type ID.
__CODE_CLASS_LOOKUP = {}

# Constructs the AST class look-up table. Used by `lookup_table` to build the
# look-up table upon its first request.
def __build_lookup_table(typ):
    if hasattr(typ, 'CODE'):
        #print("Registering AST type [%s]: %s" % (typ.CODE, typ.__name__))
        __CODE_CLASS_LOOKUP[typ.CODE] = typ
    for sub_typ in typ.__subclasses__():
        __build_lookup_table(sub_typ)

# Used to return the type ID to Python class look-up table.
# Responsible for overseeing the construction of the look-up table on its first
# invocation.
def lookup_table():
    if not __CODE_CLASS_LOOKUP:
        __build_lookup_table(Node)
    return __CODE_CLASS_LOOKUP

# The base class used by all AST nodes
class Node(object):
    # Constructs an AST node from a given JSON definition
    @staticmethod
    def from_json(jsn):
        assert 'type' in jsn, "expected 'type' property in AST node"
        typid = jsn['type']
        try:
            typ = lookup_table()[typid]
        except KeyError as e:
            raise Exception(("no Python representation of AST node with type %s found." +\
                             "Has CGum type label: %s.") % (typid, jsn['typeLabel']))
        #print("Converting AST node of (Python) type: %s" % typ.__name__)

        # Build the children of this node, if there are any, then extract any
        # attached label
        children = [Node.from_json(c) for c in jsn['children']]
        label = jsn['label'] if 'label' in jsn else None

        # Construct a node of the correct type, using the extracted information
        return typ(jsn['pos'], jsn['length'], label, children)

    def __init__(self, pos, length, label, children):
        self.__pos = int(pos)
        self.__length = int(length)
        self.__label = label
        self.__children = children
        self.__number = None
        self.__depth = None
        self.__size = None
        self.__numberStart = None
        self.__parent = None

    # Determines whether this node is equivalent to a given node.
    def equivalent(self, other):
        return  type(self) == type(other) and\
                self.__label == other.label() and\
                len(self.__children) == len(other.children()) and\
                all([c1.equivalent(c2) for (c1, c2) in zip(self.__children, other.children())])

    # By default nodes are not statements, unless overridden by another class.
    def is_statement(self):
        return False

    # Collects all nodes within the sub-tree rooted at this node which satisfy a
    # given predicate (in the form of a lambda expression)
    def collect(self, pred):
        found = [self] if pred(self) else []
        for c in self.__children:
            found += c.collect(pred)
        return found

    def parent(self):
        return self.__parent
    def label(self):
        return self.__label
    def pos(self):
        return self.__pos
    def length(self):
        return self.__length
    def children(self):
        return self.__children
    def number(self):
        return self.__number
    def numberRange(self):
        return (self.__numberStart, self.__number)
    def size(self):
        return self.__size
    def depth(self):
        return self.__depth
    def child(self, i):
        return self.__children[i]

    # Returns a list of all the descendants of this node, in order from nearest
    # to furthest.
    def descendants(self):
        return functools.reduce(lambda d, c: d + [c] + c.descendants,\
                      self.__children,\
                      [])

    # Returns a list of all the ancestors of this node, in order from nearest
    # to furthest.
    def ancestors(self):
        if self.__parent:
            return [self.__parent] + self.__parent.ancestors()
        else:
            return []

    # Returns the nearest statement to this node.
    # By default, we find the nearest statement to the parent node, if there
    # is one. If not, None is returned.
    # Statements implement a different nearestStmt, which returns the statement
    # in question.
    def nearestStmt(self):
        return self.__parent.nearestStmt() if self.__parent else None

    # Returns the index of the range that a given node belongs to, or None if
    # it belongs to none of them.
    def __find_helper(self, num, ranges, mn, mx):
        if mx < mn:
            return -1
        pivot = (mn + mx) // 2
        (num_start, num_end) = ranges[pivot]
        if num < num_start:
            return self.__find_helper(num, ranges, mn, pivot - 1)
        elif num > num_end:
            return self.__find_helper(num, ranges, pivot + 1, mx)
        else:
            return pivot

    # Finds a node with a given number within the sub-tree rooted at this node.
    def find(self, num):
        assert not num is None, "find: num must not be None"
        if self.__number == num:
            return self
        ind = self.__find_helper(num,\
                                 [c.numberRange() for c in self.__children],\
                                 0, len(self.__children) - 1)
        return self.__children[ind].find(num) if ind >= 0 else None

    def find_all(self, predicate):
        res = [self] if predicate(self) else []
        return functools.reduce(lambda r, c: r + c.find_all(predicate),\
                                self.__children, res)

    # Returns the node of the function that this node belongs to, or None if it
    # doesn't belong to a function (i.e. it's a top-level statement).

    # Pretty-prints the tree rooted at this node; useful for debugging
    def pp(self, depth=0):
        print("%s%s - %d" % ((" " * depth), self.__class__.__name__, self.__number))
        depth += 2
        for c in self.__children:
            c.pp(depth=depth)

    # Returns the CGum AST type label for this node
    def typeLabel(self):
        return self.__class__.LABEL

    # Recursively renumbers all nodes belonging to the sub-tree rooted at this
    # node. Numbers start at zero. Unfortunately necessary, since the CGum AST
    # output doesn't provide node numbers.
    def renumber(self, num=0, parent=None):
        self.__numberStart = num
        self.__parent = parent
        for c in self.__children:
            num = c.renumber(num, self)
        #print("Assigning number %d to %s at %d" % (num, self.typeLabel(), self.__pos))
        self.__number = num
        return num + 1

    def to_s(self):
        raise NotImplementedError('No `to_s` method exists for this object')
    def to_json(self):
        raise NotImplementedError('No `to_json` method exists for this object')

class Token(Node):
    def __init__(self, pos, length, label, children):
        assert not children
        assert not label
        super().__init__(pos, length, label, children)

class GenericList(Node):
    CODE = "470000"
    LABEL = "GenericList"

    def contents(self):
        return self.children()
    def __len__(self):
        return len(self.children())

class GenericString(Node):
    CODE = "480000"
    LABEL = "GenericString"

    def read(self):
        return self.label()
    def to_s(self):
        return self.label()

# :-(
class NotParsedCorrectly(Node):
    CODE = "450700"
    LABEL = "NotParsedCorrectly"

# Strangely, None can have children
class NoneNode(Node):
    CODE = "290001"
    LABEL = "None"

    def __init__(self, pos, length, label, children):
        assert label is None
        assert len(children) == 1
        assert isinstance(children[0], GenericString)
        super().__init__(pos, length, label, children)

# Equally, I have no idea what the Left node is for?
class Left(Node):
    CODE = "20100"
    LABEL = "Left"

    def __init__(self, pos, length, label, children):
        assert label is None
        assert len(children) == 1
        super().__init__(pos, length, label, children)
<commit_msg>added index_of_child method to Node<commit_after>import pprint
import functools

# This look-up table, initialised upon the first call to `from_json`, maps CGum
# AST types to their representative Python classes, using the type ID.
__CODE_CLASS_LOOKUP = {}

# Constructs the AST class look-up table. Used by `lookup_table` to build the
# look-up table upon its first request.
def __build_lookup_table(typ):
    if hasattr(typ, 'CODE'):
        #print("Registering AST type [%s]: %s" % (typ.CODE, typ.__name__))
        __CODE_CLASS_LOOKUP[typ.CODE] = typ
    for sub_typ in typ.__subclasses__():
        __build_lookup_table(sub_typ)

# Used to return the type ID to Python class look-up table.
# Responsible for overseeing the construction of the look-up table on its first
# invocation.
def lookup_table():
    if not __CODE_CLASS_LOOKUP:
        __build_lookup_table(Node)
    return __CODE_CLASS_LOOKUP

# The base class used by all AST nodes
class Node(object):
    # Constructs an AST node from a given JSON definition
    @staticmethod
    def from_json(jsn):
        assert 'type' in jsn, "expected 'type' property in AST node"
        typid = jsn['type']
        try:
            typ = lookup_table()[typid]
        except KeyError as e:
            raise Exception(("no Python representation of AST node with type %s found." +\
                             "Has CGum type label: %s.") % (typid, jsn['typeLabel']))
        #print("Converting AST node of (Python) type: %s" % typ.__name__)

        # Build the children of this node, if there are any, then extract any
        # attached label
        children = [Node.from_json(c) for c in jsn['children']]
        label = jsn['label'] if 'label' in jsn else None

        # Construct a node of the correct type, using the extracted information
        return typ(jsn['pos'], jsn['length'], label, children)

    def __init__(self, pos, length, label, children):
        self.__pos = int(pos)
        self.__length = int(length)
        self.__label = label
        self.__children = children
        self.__number = None
        self.__depth = None
        self.__size = None
        self.__numberStart = None
        self.__parent = None

    # Determines whether this node is equivalent to a given node.
    def equivalent(self, other):
        return  type(self) == type(other) and\
                self.__label == other.label() and\
                len(self.__children) == len(other.children()) and\
                all([c1.equivalent(c2) for (c1, c2) in zip(self.__children, other.children())])

    # By default nodes are not statements, unless overridden by another class.
    def is_statement(self):
        return False

    # Collects all nodes within the sub-tree rooted at this node which satisfy a
    # given predicate (in the form of a lambda expression)
    def collect(self, pred):
        found = [self] if pred(self) else []
        for c in self.__children:
            found += c.collect(pred)
        return found

    def parent(self):
        return self.__parent
    def label(self):
        return self.__label
    def pos(self):
        return self.__pos
    def length(self):
        return self.__length
    def children(self):
        return self.__children
    def number(self):
        return self.__number
    def numberRange(self):
        return (self.__numberStart, self.__number)
    def size(self):
        return self.__size
    def depth(self):
        return self.__depth
    def child(self, i):
        return self.__children[i]

    # Returns the index of a given child of this node.
    # If the given node is not a child of this node, returns None.
    # Throws an assertion error if the child is None.
    def index_of_child(self, child):
        assert not child is None, "indexOfChild: child must not be None"
        for (i, c) in enumerate(self.children()):
            if c == child:
                return i
        return None

    # Returns a list of all the descendants of this node, in order from nearest
    # to furthest.
    def descendants(self):
        return functools.reduce(lambda d, c: d + [c] + c.descendants,\
                      self.__children,\
                      [])

    # Returns a list of all the ancestors of this node, in order from nearest
    # to furthest.
    def ancestors(self):
        if self.__parent:
            return [self.__parent] + self.__parent.ancestors()
        else:
            return []

    # Returns the nearest statement to this node.
    # By default, we find the nearest statement to the parent node, if there
    # is one. If not, None is returned.
    # Statements implement a different nearestStmt, which returns the statement
    # in question.
    def nearestStmt(self):
        return self.__parent.nearestStmt() if self.__parent else None

    # Returns the index of the range that a given node belongs to, or None if
    # it belongs to none of them.
    def __find_helper(self, num, ranges, mn, mx):
        if mx < mn:
            return -1
        pivot = (mn + mx) // 2
        (num_start, num_end) = ranges[pivot]
        if num < num_start:
            return self.__find_helper(num, ranges, mn, pivot - 1)
        elif num > num_end:
            return self.__find_helper(num, ranges, pivot + 1, mx)
        else:
            return pivot

    # Finds a node with a given number within the sub-tree rooted at this node.
    def find(self, num):
        assert not num is None, "find: num must not be None"
        if self.__number == num:
            return self
        ind = self.__find_helper(num,\
                                 [c.numberRange() for c in self.__children],\
                                 0, len(self.__children) - 1)
        return self.__children[ind].find(num) if ind >= 0 else None

    def find_all(self, predicate):
        res = [self] if predicate(self) else []
        return functools.reduce(lambda r, c: r + c.find_all(predicate),\
                                self.__children, res)

    # Returns the node of the function that this node belongs to, or None if it
    # doesn't belong to a function (i.e. it's a top-level statement).

    # Pretty-prints the tree rooted at this node; useful for debugging
    def pp(self, depth=0):
        print("%s%s - %d" % ((" " * depth), self.__class__.__name__, self.__number))
        depth += 2
        for c in self.__children:
            c.pp(depth=depth)

    # Returns the CGum AST type label for this node
    def typeLabel(self):
        return self.__class__.LABEL

    # Recursively renumbers all nodes belonging to the sub-tree rooted at this
    # node. Numbers start at zero. Unfortunately necessary, since the CGum AST
    # output doesn't provide node numbers.
    def renumber(self, num=0, parent=None):
        self.__numberStart = num
        self.__parent = parent
        for c in self.__children:
            num = c.renumber(num, self)
        #print("Assigning number %d to %s at %d" % (num, self.typeLabel(), self.__pos))
        self.__number = num
        return num + 1

    def to_s(self):
        raise NotImplementedError('No `to_s` method exists for this object')
    def to_json(self):
        raise NotImplementedError('No `to_json` method exists for this object')

class Token(Node):
    def __init__(self, pos, length, label, children):
        assert not children
        assert not label
        super().__init__(pos, length, label, children)

class GenericList(Node):
    CODE = "470000"
    LABEL = "GenericList"

    def contents(self):
        return self.children()
    def __len__(self):
        return len(self.children())

class GenericString(Node):
    CODE = "480000"
    LABEL = "GenericString"

    def read(self):
        return self.label()
    def to_s(self):
        return self.label()

# :-(
class NotParsedCorrectly(Node):
    CODE = "450700"
    LABEL = "NotParsedCorrectly"

# Strangely, None can have children
class NoneNode(Node):
    CODE = "290001"
    LABEL = "None"

    def __init__(self, pos, length, label, children):
        assert label is None
        assert len(children) == 1
        assert isinstance(children[0], GenericString)
        super().__init__(pos, length, label, children)

# Equally, I have no idea what the Left node is for?
class Left(Node):
    CODE = "20100"
    LABEL = "Left"

    def __init__(self, pos, length, label, children):
        assert label is None
        assert len(children) == 1
        super().__init__(pos, length, label, children)
<|endoftext|>


<commit_before>import os
import sys
import shutil
import logging
import collections

if os.environ.has_key("WAFDIR"):
    WAFDIR = os.path.join(os.environ["WAFDIR"], "waflib")
else:
    WAFDIR = os.path.join(os.getcwd(), "waflib")
if not os.path.exists(WAFDIR):
    raise RuntimeError("%r not found: required when using waf extras !" % WAFDIR)
sys.path.insert(0, os.path.dirname(WAFDIR))

from waflib.Context \
    import \
        create_context
from waflib.Options \
    import \
        OptionsContext
from waflib import Options
from waflib import Context
from waflib import Logs
from waflib import Build
from waflib import Utils
import waflib

from bento.core.node_package \
    import \
        translate_name
from bento.commands.options \
    import \
        Option
from bento.commands.context \
    import \
        ConfigureContext, BuildContext
from bento.installed_package_description \
    import \
        InstalledSection
from bento.core.utils \
    import \
        normalize_path

WAF_TOP = os.path.join(WAFDIR, os.pardir)

WAF_CONFIG_LOG = 'config.log'

__USE_NO_OUTPUT_LOGGING = False
def disable_output():
    # Make Betty proud...
    global __USE_NO_OUTPUT_LOGGING
    __USE_NO_OUTPUT_LOGGING = True

def make_stream_logger(name, stream):
    # stream should be a file-like object supporting write/read/?
    logger = logging.getLogger(name)
    hdlr = logging.StreamHandler(stream)
    formatter = logging.Formatter('%(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    logger.setLevel(logging.DEBUG)
    return logger

def _init_log_no_output():
    # XXX: all this heavily plays with waf internals - only use for unit
    # testing bento where waf output screws up nose own output/logging magic
    from cStringIO import StringIO
    Logs.got_tty = False
    Logs.get_term_cols = lambda: 80
    Logs.get_color = lambda cl: ''
    Logs.colors = Logs.color_dict()

    fake_output = StringIO()
    def fake_pprint(col, str, label='', sep='\n'):
        fake_output.write("%s%s%s %s%s" % (Logs.colors(col), str, Logs.colors.NORMAL, label, sep))

    Logs.pprint = fake_pprint

    log = logging.getLogger('waflib')
    log.handlers = []
    log.filters = []
    hdlr = logging.StreamHandler(StringIO())
    hdlr.setFormatter(Logs.formatter())
    log.addHandler(hdlr)
    log.addFilter(Logs.log_filter())
    log.setLevel(logging.DEBUG)
    Logs.log = log

def _init(run_path, source_path, build_path):
    if not (os.path.isabs(run_path) and os.path.isabs(source_path) and os.path.isabs(build_path)):
        raise ValueError("All paths must be absolute !")
    tooldir = os.path.join(WAFDIR, "Tools")

    sys.path.insert(0, tooldir)
    cwd = os.getcwd()

    if __USE_NO_OUTPUT_LOGGING is True:
        _init_log_no_output()
    else:
        Logs.init_log()

    class FakeModule(object):
        pass
    Context.g_module = FakeModule
    Context.g_module.root_path = os.path.abspath(__file__)
    Context.g_module.top = source_path
    Context.g_module.out = build_path

    Context.top_dir = source_path
    Context.run_dir = run_path
    Context.out_dir = build_path
    Context.waf_dir = WAF_TOP

def register_options(global_context):
    opt = Option("-p", "--progress", help="Use progress bar", action="store_true", dest="progress_bar")
    global_context.add_option("build", opt)

class ConfigureWafContext(ConfigureContext):
    def __init__(self, cmd_argv, options_context, pkg, run_node):
        super(ConfigureWafContext, self).__init__(cmd_argv, options_context, pkg, run_node)

        run_path = self.run_node.abspath()
        source_path = self.top_node.abspath()
        build_path = self.build_node.abspath()
        _init(run_path=run_path, source_path=source_path, build_path=build_path)

        opts = OptionsContext()
        opts.parse_args([])
        opts.load("compiler_c")
        Options.options.check_c_compiler = "gcc"
        self.waf_options_context = opts

        waf_context = create_context("configure", run_dir=source_path)
        waf_context.options = Options.options
        waf_context.init_dirs()
        waf_context.cachedir = waf_context.bldnode.make_node(Build.CACHE_DIR)
        waf_context.cachedir.mkdir()

        path = os.path.join(waf_context.bldnode.abspath(), WAF_CONFIG_LOG)
        waf_context.logger = Logs.make_logger(path, 'cfg')
        self.waf_context = waf_context

        # FIXME: this is wrong (not taking into account sub packages)
        has_compiled_code = len(pkg.extensions) > 0 or len(pkg.compiled_libraries) > 0
        if not has_compiled_code:
            if pkg.subpackages:
                for v in pkg.subpackages.values():
                    if len(v.extensions) > 0 or len(v.compiled_libraries) > 0:
                        has_compiled_code = True
                        break
        conf = self.waf_context
        if has_compiled_code:
            conf.load("compiler_c")
            conf.load("python")
            conf.check_python_version((2,4,2))
            conf.check_python_headers()

            # HACK for mac os x
            if sys.platform == "darwin":
                conf.env["CC"] = ["/usr/bin/gcc-4.0"]
        self._old_path = None

    def pre_recurse(self, local_node):
        ConfigureContext.pre_recurse(self, local_node)
        self._old_path = self.waf_context.path
        # Gymnastic to make a *waf* node from a *bento* node
        self.waf_context.path = self.waf_context.path.make_node(self.local_node.srcpath())

    def post_recurse(self):
        self.waf_context.path = self._old_path
        ConfigureContext.post_recurse(self)

    def shutdown(self):
        super(ConfigureWafContext, self).shutdown()
        self.waf_context.store()

def ext_name_to_path(name):
    """Convert extension name to path - the path does not include the
    file extension

    Example: foo.bar -> foo/bar
    """
    return name.replace('.', os.path.sep)

class BentoBuildContext(Build.BuildContext):
    """Waf build context with additional support to register builder output to
    bento build context."""
    def __init__(self, *a, **kw):
        Build.BuildContext.__init__(self, *a, **kw)
        # XXX: set into BuildWafContext
        self.bento_context = None

    def register_outputs(self, category, name, outputs):
        outputs_registry = self.bento_context.outputs_registry

        # waf -> bento nodes translations
        nodes = [self.bento_context.build_node.make_node(n.bldpath()) for n in outputs]
        from_node = self.bento_context.build_node.find_node(outputs[0].parent.bldpath())

        pkg_dir = os.path.dirname(name.replace('.', os.path.sep))
        target_dir = os.path.join('$sitedir', pkg_dir)
        outputs_registry.register_outputs(category, name, nodes, from_node, target_dir)

@waflib.TaskGen.feature("bento")
@waflib.TaskGen.before_method("apply_link")
def apply_pyext_target_renaming(self):
    if not self.target:
        self.target = self.name
    self.target = self.target.replace(".", os.sep)

@waflib.TaskGen.feature("bento")
@waflib.TaskGen.after_method("apply_link")
def apply_register_outputs(self):
    for x in self.features:
        if x == "cprogram" and "cxx" in self.features:
            x = "cxxprogram"
        if x == "cshlib" and "cxx" in self.features:
            x = "cxxshlib"

        if x in waflib.Task.classes:
            if issubclass(waflib.Task.classes[x], waflib.Tools.ccroot.link_task):
                link = x
                break
    else:
        return

    if "pyext" in self.features and "cshlib" in self.features:
        category = "extensions"
    else:
        category = "compiled_libraries"
    bento_context = self.bld.bento_context
    ref_node = bento_context.top_node.make_node(self.path.path_from(self.path.ctx.srcnode))
    name = translate_name(self.name, ref_node, bento_context.top_node)
    self.bld.register_outputs(category, name, self.link_task.outputs)

class BuildWafContext(BuildContext):
    def pre_recurse(self, local_node):
        super(BuildWafContext, self).pre_recurse(local_node)
        self._old_path = self.waf_context.path
        # Gymnastic to make a *waf* node from a *bento* node
        self.waf_context.path = self.waf_context.path.make_node(self.local_node.srcpath())

    def post_recurse(self):
        self.waf_context.path = self._old_path
        super(BuildWafContext, self).post_recurse()

    def __init__(self, cmd_argv, options_context, pkg, run_node):
        super(BuildWafContext, self).__init__(cmd_argv, options_context, pkg, run_node)

        o, a = options_context.parser.parse_args(cmd_argv)
        if o.jobs:
            jobs = int(o.jobs)
        else:
            jobs = 1
        if o.verbose:
            verbose = int(o.verbose)
            zones = ["runner"]
        else:
            verbose = 0
            zones = []
        if o.inplace:
            self.inplace = 1
        else:
            self.inplace = 0
        if o.progress_bar:
            self.progress_bar = True
        else:
            self.progress_bar = False

        Logs.verbose = verbose
        Logs.init_log()
        if zones is None:
            Logs.zones = []
        else:
            Logs.zones = zones

        run_path = self.run_node.abspath()
        source_path = self.top_node.abspath()
        build_path = self.build_node.abspath()
        _init(run_path=run_path, source_path=source_path, build_path=build_path)
        waf_context = create_context("build")
        waf_context.restore()
        if not waf_context.all_envs:
            waf_context.load_envs()
        waf_context.jobs = jobs
        waf_context.timer = Utils.Timer()
        if self.progress_bar:
            waf_context.progress_bar = 1
        waf_context.bento_context = self
        self.waf_context = waf_context

        def _default_extension_builder(extension, **kw):
            if not "features" in kw:
                kw["features"] = "c cloadable pyext bento"
            if not "source" in kw:
                kw["source"] = extension.sources[:]
            if not "name" in kw:
                kw["name"] = extension.name
            return self.waf_context(**kw)

        def _default_library_builder(library):
            return self.waf_context(features='c cstlib pyext bento', source=library.sources,
                                    name=library.name)

        self.builder_registry.register_category("extensions", _default_extension_builder)
        self.builder_registry.register_category("compiled_libraries", _default_library_builder)

    def compile(self):
        super(BuildWafContext, self).compile()
        reg = self.builder_registry

        for category in ("extensions", "compiled_libraries"):
            for name, extension in self._node_pkg.iter_category(category):
                builder = reg.builder(category, name)
                self.pre_recurse(extension.ref_node)
                try:
                    extension = extension.extension_from(extension.ref_node)
                    task_gen = builder(extension)
                finally:
                    self.post_recurse()

        if self.progress_bar:
            sys.stderr.write(Logs.colors.cursor_off)
        try:
            self.waf_context.compile()
        finally:
            if self.progress_bar:
                c = len(self.waf_context.returned_tasks) or 1
                self.waf_context.to_log(self.waf_context.progress_line(c, c, Logs.colors.BLUE, Logs.colors.NORMAL))
                print('')
                sys.stdout.flush()
                sys.stderr.write(Logs.colors.cursor_on)

    def shutdown(self):
        super(BuildWafContext, self).shutdown()
        self.waf_context.store()
<commit_msg>ENH: update waf default library builder to follow extension builder.<commit_after>import os
import sys
import shutil
import logging
import collections

if os.environ.has_key("WAFDIR"):
    WAFDIR = os.path.join(os.environ["WAFDIR"], "waflib")
else:
    WAFDIR = os.path.join(os.getcwd(), "waflib")
if not os.path.exists(WAFDIR):
    raise RuntimeError("%r not found: required when using waf extras !" % WAFDIR)
sys.path.insert(0, os.path.dirname(WAFDIR))

from waflib.Context \
    import \
        create_context
from waflib.Options \
    import \
        OptionsContext
from waflib import Options
from waflib import Context
from waflib import Logs
from waflib import Build
from waflib import Utils
import waflib

from bento.core.node_package \
    import \
        translate_name
from bento.commands.options \
    import \
        Option
from bento.commands.context \
    import \
        ConfigureContext, BuildContext
from bento.installed_package_description \
    import \
        InstalledSection
from bento.core.utils \
    import \
        normalize_path

WAF_TOP = os.path.join(WAFDIR, os.pardir)

WAF_CONFIG_LOG = 'config.log'

__USE_NO_OUTPUT_LOGGING = False
def disable_output():
    # Make Betty proud...
    global __USE_NO_OUTPUT_LOGGING
    __USE_NO_OUTPUT_LOGGING = True

def make_stream_logger(name, stream):
    # stream should be a file-like object supporting write/read/?
    logger = logging.getLogger(name)
    hdlr = logging.StreamHandler(stream)
    formatter = logging.Formatter('%(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    logger.setLevel(logging.DEBUG)
    return logger

def _init_log_no_output():
    # XXX: all this heavily plays with waf internals - only use for unit
    # testing bento where waf output screws up nose own output/logging magic
    from cStringIO import StringIO
    Logs.got_tty = False
    Logs.get_term_cols = lambda: 80
    Logs.get_color = lambda cl: ''
    Logs.colors = Logs.color_dict()

    fake_output = StringIO()
    def fake_pprint(col, str, label='', sep='\n'):
        fake_output.write("%s%s%s %s%s" % (Logs.colors(col), str, Logs.colors.NORMAL, label, sep))

    Logs.pprint = fake_pprint

    log = logging.getLogger('waflib')
    log.handlers = []
    log.filters = []
    hdlr = logging.StreamHandler(StringIO())
    hdlr.setFormatter(Logs.formatter())
    log.addHandler(hdlr)
    log.addFilter(Logs.log_filter())
    log.setLevel(logging.DEBUG)
    Logs.log = log

def _init(run_path, source_path, build_path):
    if not (os.path.isabs(run_path) and os.path.isabs(source_path) and os.path.isabs(build_path)):
        raise ValueError("All paths must be absolute !")
    tooldir = os.path.join(WAFDIR, "Tools")

    sys.path.insert(0, tooldir)
    cwd = os.getcwd()

    if __USE_NO_OUTPUT_LOGGING is True:
        _init_log_no_output()
    else:
        Logs.init_log()

    class FakeModule(object):
        pass
    Context.g_module = FakeModule
    Context.g_module.root_path = os.path.abspath(__file__)
    Context.g_module.top = source_path
    Context.g_module.out = build_path

    Context.top_dir = source_path
    Context.run_dir = run_path
    Context.out_dir = build_path
    Context.waf_dir = WAF_TOP

def register_options(global_context):
    opt = Option("-p", "--progress", help="Use progress bar", action="store_true", dest="progress_bar")
    global_context.add_option("build", opt)

class ConfigureWafContext(ConfigureContext):
    def __init__(self, cmd_argv, options_context, pkg, run_node):
        super(ConfigureWafContext, self).__init__(cmd_argv, options_context, pkg, run_node)

        run_path = self.run_node.abspath()
        source_path = self.top_node.abspath()
        build_path = self.build_node.abspath()
        _init(run_path=run_path, source_path=source_path, build_path=build_path)

        opts = OptionsContext()
        opts.parse_args([])
        opts.load("compiler_c")
        Options.options.check_c_compiler = "gcc"
        self.waf_options_context = opts

        waf_context = create_context("configure", run_dir=source_path)
        waf_context.options = Options.options
        waf_context.init_dirs()
        waf_context.cachedir = waf_context.bldnode.make_node(Build.CACHE_DIR)
        waf_context.cachedir.mkdir()

        path = os.path.join(waf_context.bldnode.abspath(), WAF_CONFIG_LOG)
        waf_context.logger = Logs.make_logger(path, 'cfg')
        self.waf_context = waf_context

        # FIXME: this is wrong (not taking into account sub packages)
        has_compiled_code = len(pkg.extensions) > 0 or len(pkg.compiled_libraries) > 0
        if not has_compiled_code:
            if pkg.subpackages:
                for v in pkg.subpackages.values():
                    if len(v.extensions) > 0 or len(v.compiled_libraries) > 0:
                        has_compiled_code = True
                        break
        conf = self.waf_context
        if has_compiled_code:
            conf.load("compiler_c")
            conf.load("python")
            conf.check_python_version((2,4,2))
            conf.check_python_headers()

            # HACK for mac os x
            if sys.platform == "darwin":
                conf.env["CC"] = ["/usr/bin/gcc-4.0"]
        self._old_path = None

    def pre_recurse(self, local_node):
        ConfigureContext.pre_recurse(self, local_node)
        self._old_path = self.waf_context.path
        # Gymnastic to make a *waf* node from a *bento* node
        self.waf_context.path = self.waf_context.path.make_node(self.local_node.srcpath())

    def post_recurse(self):
        self.waf_context.path = self._old_path
        ConfigureContext.post_recurse(self)

    def shutdown(self):
        super(ConfigureWafContext, self).shutdown()
        self.waf_context.store()

def ext_name_to_path(name):
    """Convert extension name to path - the path does not include the
    file extension

    Example: foo.bar -> foo/bar
    """
    return name.replace('.', os.path.sep)

class BentoBuildContext(Build.BuildContext):
    """Waf build context with additional support to register builder output to
    bento build context."""
    def __init__(self, *a, **kw):
        Build.BuildContext.__init__(self, *a, **kw)
        # XXX: set into BuildWafContext
        self.bento_context = None

    def register_outputs(self, category, name, outputs):
        outputs_registry = self.bento_context.outputs_registry

        # waf -> bento nodes translations
        nodes = [self.bento_context.build_node.make_node(n.bldpath()) for n in outputs]
        from_node = self.bento_context.build_node.find_node(outputs[0].parent.bldpath())

        pkg_dir = os.path.dirname(name.replace('.', os.path.sep))
        target_dir = os.path.join('$sitedir', pkg_dir)
        outputs_registry.register_outputs(category, name, nodes, from_node, target_dir)

@waflib.TaskGen.feature("bento")
@waflib.TaskGen.before_method("apply_link")
def apply_pyext_target_renaming(self):
    if not self.target:
        self.target = self.name
    self.target = self.target.replace(".", os.sep)

@waflib.TaskGen.feature("bento")
@waflib.TaskGen.after_method("apply_link")
def apply_register_outputs(self):
    for x in self.features:
        if x == "cprogram" and "cxx" in self.features:
            x = "cxxprogram"
        if x == "cshlib" and "cxx" in self.features:
            x = "cxxshlib"

        if x in waflib.Task.classes:
            if issubclass(waflib.Task.classes[x], waflib.Tools.ccroot.link_task):
                link = x
                break
    else:
        return

    if "pyext" in self.features and "cshlib" in self.features:
        category = "extensions"
    else:
        category = "compiled_libraries"
    bento_context = self.bld.bento_context
    ref_node = bento_context.top_node.make_node(self.path.path_from(self.path.ctx.srcnode))
    name = translate_name(self.name, ref_node, bento_context.top_node)
    self.bld.register_outputs(category, name, self.link_task.outputs)

class BuildWafContext(BuildContext):
    def pre_recurse(self, local_node):
        super(BuildWafContext, self).pre_recurse(local_node)
        self._old_path = self.waf_context.path
        # Gymnastic to make a *waf* node from a *bento* node
        self.waf_context.path = self.waf_context.path.make_node(self.local_node.srcpath())

    def post_recurse(self):
        self.waf_context.path = self._old_path
        super(BuildWafContext, self).post_recurse()

    def __init__(self, cmd_argv, options_context, pkg, run_node):
        super(BuildWafContext, self).__init__(cmd_argv, options_context, pkg, run_node)

        o, a = options_context.parser.parse_args(cmd_argv)
        if o.jobs:
            jobs = int(o.jobs)
        else:
            jobs = 1
        if o.verbose:
            verbose = int(o.verbose)
            zones = ["runner"]
        else:
            verbose = 0
            zones = []
        if o.inplace:
            self.inplace = 1
        else:
            self.inplace = 0
        if o.progress_bar:
            self.progress_bar = True
        else:
            self.progress_bar = False

        Logs.verbose = verbose
        Logs.init_log()
        if zones is None:
            Logs.zones = []
        else:
            Logs.zones = zones

        run_path = self.run_node.abspath()
        source_path = self.top_node.abspath()
        build_path = self.build_node.abspath()
        _init(run_path=run_path, source_path=source_path, build_path=build_path)
        waf_context = create_context("build")
        waf_context.restore()
        if not waf_context.all_envs:
            waf_context.load_envs()
        waf_context.jobs = jobs
        waf_context.timer = Utils.Timer()
        if self.progress_bar:
            waf_context.progress_bar = 1
        waf_context.bento_context = self
        self.waf_context = waf_context

        def _default_extension_builder(extension, **kw):
            if not "features" in kw:
                kw["features"] = "c cloadable pyext bento"
            if not "source" in kw:
                kw["source"] = extension.sources[:]
            if not "name" in kw:
                kw["name"] = extension.name
            return self.waf_context(**kw)

        def _default_library_builder(library, **kw):
            if not "features" in kw:
                kw["features"] = "c cstlib pyext bento"
            if not "source" in kw:
                kw["source"] = library.sources[:]
            if not "name" in kw:
                kw["name"] = library.name
            return self.waf_context(**kw)

        self.builder_registry.register_category("extensions", _default_extension_builder)
        self.builder_registry.register_category("compiled_libraries", _default_library_builder)

    def compile(self):
        super(BuildWafContext, self).compile()
        reg = self.builder_registry

        for category in ("extensions", "compiled_libraries"):
            for name, extension in self._node_pkg.iter_category(category):
                builder = reg.builder(category, name)
                self.pre_recurse(extension.ref_node)
                try:
                    extension = extension.extension_from(extension.ref_node)
                    task_gen = builder(extension)
                finally:
                    self.post_recurse()

        if self.progress_bar:
            sys.stderr.write(Logs.colors.cursor_off)
        try:
            self.waf_context.compile()
        finally:
            if self.progress_bar:
                c = len(self.waf_context.returned_tasks) or 1
                self.waf_context.to_log(self.waf_context.progress_line(c, c, Logs.colors.BLUE, Logs.colors.NORMAL))
                print('')
                sys.stdout.flush()
                sys.stderr.write(Logs.colors.cursor_on)

    def shutdown(self):
        super(BuildWafContext, self).shutdown()
        self.waf_context.store()
<|endoftext|>


<commit_before>import os
import re
import json

from os.path import dirname
from os.path import abspath
from os.path import join as pathjoin
from os.path import basename

from .bids_validator import BIDSValidator
from grabbit import Layout

__all__ = ['BIDSLayout']

class BIDSLayout(Layout):
    def __init__(self, path, config=None, validate=False, **kwargs):
        self.validator = BIDSValidator()
        self.validate = validate
        self.project_path = os.path.abspath(path)
        if config is None:
            root = dirname(abspath(__file__))
            config = pathjoin(root, 'config', 'bids.json')
        super(BIDSLayout, self).__init__(path, config,
                                         dynamic_getters=True, **kwargs)

    def _validate_file(self, f):
        to_check = f.path
        to_check = to_check.split(self.project_path, maxsplit=1)[1]

        if to_check[0] != '/':
            to_check = '/' + to_check
        else:
            None
        print(to_check)
        return (not self.validate) or self.validator.is_bids(to_check)

    def get_nearest_helper(self, path, extension, type=None, **kwargs):
        path = abspath(path)

        if path not in self.files:
            raise ValueError("File '%s' could not be found in the current BIDS"
                             " project." % path)

        if not type:
            # Constrain the search to .json files with the same type as target
            type = self.files[path].entities['type']

        return self.get_nearest(path, extensions=extension, all_=True,
                                type=type, ignore_strict_entities=['type'],
                                **kwargs)

    def get_metadata(self, path, **kwargs):

        potentialJSONs = self.get_nearest_helper(path, '.json', **kwargs)

        merged_param_dict = {}
        for json_file_path in reversed(potentialJSONs):
            if os.path.exists(json_file_path):
                param_dict = json.load(open(json_file_path, "r"))
                merged_param_dict.update(param_dict)

        return merged_param_dict

    def get_bvec(self, path, **kwargs):
        return self.get_nearest_helper(path, '.bvec', **kwargs)[0]

    def get_bval(self, path, **kwargs):
        return self.get_nearest_helper(path, '.bval', **kwargs)[0]

    def get_events(self, path, **kwargs):
        return self.get_nearest_helper(path, '.tsv', 'events', **kwargs)[0]

    def get_fieldmap(self, path, return_list=False):
        fieldmaps = self._get_fieldmaps(path)

        if return_list:
            return fieldmaps
        else:
            if len(fieldmaps) == 1:
                return fieldmaps[0]
            elif len(fieldmaps) > 1:
                raise ValueError("More than one fieldmap found, but the "
                                 "'return_list' argument was set to False. "
                                 "Either ensure that there is only one "
                                 "fieldmap for this image, or set the "
                                 "'return_list' argument to True and handle "
                                 "the result as a list.")
            else:  # len(fieldmaps) == 0
                return None

    def _get_fieldmaps(self, path):
        sub = os.path.split(path)[1].split("_")[0].split("sub-")[1]
        fieldmap_set = []
        type_ = '(phase1|phasediff|epi|fieldmap)'
        for file in self.get(subject=sub, type=type_,
                             extensions=['nii.gz', 'nii']):
            metadata = self.get_metadata(file.filename)
            if metadata and "IntendedFor" in metadata.keys():
                if isinstance(metadata["IntendedFor"], list):
                    intended_for = metadata["IntendedFor"]
                else:
                    intended_for = [metadata["IntendedFor"]]
                if any([path.endswith(suffix) for suffix in intended_for]):
                    cur_fieldmap = {}
                    if file.type == "phasediff":
                        cur_fieldmap = {"phasediff": file.filename,
                                        "magnitude1": file.filename.replace(
                                            "phasediff", "magnitude1"),
                                        "magnitude2": file.filename.replace(
                                            "phasediff", "magnitude2"),
                                        "type": "phasediff"}
                    elif file.type == "phase1":
                        cur_fieldmap["phase1"] = file.filename
                        cur_fieldmap["magnitude1"] = \
                            file.filename.replace("phase1", "magnitude1")
                        cur_fieldmap["phase2"] = \
                            file.filename.replace("phase1", "phase2")
                        cur_fieldmap["magnitude2"] = \
                            file.filename.replace("phase1", "magnitude2")
                        cur_fieldmap["type"] = "phase"
                    elif file.type == "epi":
                        cur_fieldmap["epi"] = file.filename
                        cur_fieldmap["type"] = "epi"
                    elif file.type == "fieldmap":
                        cur_fieldmap["fieldmap"] = file.filename
                        cur_fieldmap["magnitude"] = \
                            file.filename.replace("fieldmap", "magnitude")
                        cur_fieldmap["type"] = "fieldmap"
                    fieldmap_set.append(cur_fieldmap)
        return fieldmap_set

    def find_match(self, target, source=None):

        # Try to take the easy way out
        if source is not None:
            _target = source.split('.')[0] + '.' + target
            if os.path.exists(_target):
                return target

        if target in list(self.entities.keys()):
            candidates = list(self.entities[target].files.keys())
        else:
            candidates = []

            for root, directories, filenames in os.walk(self.root):
                for f in filenames:
                    if re.search(target + '$', f):
                        if os.path.sep == "\\":
                            f = f.replace("\\", "\\\\")
                        candidates.append(f)

        if source is None:
            return candidates

        # Walk up the file hierarchy from source, find first match
        if not os.path.exists(source):
            raise OSError("The file '%s' doesn't exist." % source)
        elif not source.startswith(self.root):
            raise ValueError("The file '%s' is not contained "
                             "within the current project "
                             "directory (%s)." % (source, self.root))
        rel = os.path.relpath(dirname(source), self.root)
        sep = os.path.sep
        chunks = rel.split(sep)
        n_chunks = len(chunks)
        for i in range(n_chunks, -1, -1):
            path = pathjoin(self.root, *chunks[:i])
            patt = path + '\%s[^\%s]+$' % (sep, sep)
            if sep == "\\":
                patt = path + '\\[^\\]+$'
                patt = patt.replace("\\", "\\\\")
            matches = [x for x in candidates if re.search(patt, x)]
            if matches:
                if len(matches) == 1:
                    return matches[0]
                else:
                    raise ValueError("Ambiguous target: more than one "
                                     "candidate file found in "
                                     "directory '%s'." % path)
        return None
<commit_msg>reworked validate function to avoid unnecessary calculations<commit_after>import os
import re
import json

from os.path import dirname
from os.path import abspath
from os.path import join as pathjoin
from os.path import basename

from .bids_validator import BIDSValidator
from grabbit import Layout

__all__ = ['BIDSLayout']

class BIDSLayout(Layout):
    def __init__(self, path, config=None, validate=False, **kwargs):
        self.validator = BIDSValidator()
        self.validate = validate
        self.project_path = os.path.abspath(path)
        if config is None:
            root = dirname(abspath(__file__))
            config = pathjoin(root, 'config', 'bids.json')
        super(BIDSLayout, self).__init__(path, config,
                                         dynamic_getters=True, **kwargs)

    def _validate_file(self, f):
        if not self.validate:
            return True
        to_check = f.path
        to_check = to_check.split(self.project_path, maxsplit=1)[1]

        if to_check[0] != '/':
            to_check = '/' + to_check
        else:
            None
        print(to_check)
        return self.validator.is_bids(to_check)

    def get_nearest_helper(self, path, extension, type=None, **kwargs):
        path = abspath(path)

        if path not in self.files:
            raise ValueError("File '%s' could not be found in the current BIDS"
                             " project." % path)

        if not type:
            # Constrain the search to .json files with the same type as target
            type = self.files[path].entities['type']

        return self.get_nearest(path, extensions=extension, all_=True,
                                type=type, ignore_strict_entities=['type'],
                                **kwargs)

    def get_metadata(self, path, **kwargs):

        potentialJSONs = self.get_nearest_helper(path, '.json', **kwargs)

        merged_param_dict = {}
        for json_file_path in reversed(potentialJSONs):
            if os.path.exists(json_file_path):
                param_dict = json.load(open(json_file_path, "r"))
                merged_param_dict.update(param_dict)

        return merged_param_dict

    def get_bvec(self, path, **kwargs):
        return self.get_nearest_helper(path, '.bvec', **kwargs)[0]

    def get_bval(self, path, **kwargs):
        return self.get_nearest_helper(path, '.bval', **kwargs)[0]

    def get_events(self, path, **kwargs):
        return self.get_nearest_helper(path, '.tsv', 'events', **kwargs)[0]

    def get_fieldmap(self, path, return_list=False):
        fieldmaps = self._get_fieldmaps(path)

        if return_list:
            return fieldmaps
        else:
            if len(fieldmaps) == 1:
                return fieldmaps[0]
            elif len(fieldmaps) > 1:
                raise ValueError("More than one fieldmap found, but the "
                                 "'return_list' argument was set to False. "
                                 "Either ensure that there is only one "
                                 "fieldmap for this image, or set the "
                                 "'return_list' argument to True and handle "
                                 "the result as a list.")
            else:  # len(fieldmaps) == 0
                return None

    def _get_fieldmaps(self, path):
        sub = os.path.split(path)[1].split("_")[0].split("sub-")[1]
        fieldmap_set = []
        type_ = '(phase1|phasediff|epi|fieldmap)'
        for file in self.get(subject=sub, type=type_,
                             extensions=['nii.gz', 'nii']):
            metadata = self.get_metadata(file.filename)
            if metadata and "IntendedFor" in metadata.keys():
                if isinstance(metadata["IntendedFor"], list):
                    intended_for = metadata["IntendedFor"]
                else:
                    intended_for = [metadata["IntendedFor"]]
                if any([path.endswith(suffix) for suffix in intended_for]):
                    cur_fieldmap = {}
                    if file.type == "phasediff":
                        cur_fieldmap = {"phasediff": file.filename,
                                        "magnitude1": file.filename.replace(
                                            "phasediff", "magnitude1"),
                                        "magnitude2": file.filename.replace(
                                            "phasediff", "magnitude2"),
                                        "type": "phasediff"}
                    elif file.type == "phase1":
                        cur_fieldmap["phase1"] = file.filename
                        cur_fieldmap["magnitude1"] = \
                            file.filename.replace("phase1", "magnitude1")
                        cur_fieldmap["phase2"] = \
                            file.filename.replace("phase1", "phase2")
                        cur_fieldmap["magnitude2"] = \
                            file.filename.replace("phase1", "magnitude2")
                        cur_fieldmap["type"] = "phase"
                    elif file.type == "epi":
                        cur_fieldmap["epi"] = file.filename
                        cur_fieldmap["type"] = "epi"
                    elif file.type == "fieldmap":
                        cur_fieldmap["fieldmap"] = file.filename
                        cur_fieldmap["magnitude"] = \
                            file.filename.replace("fieldmap", "magnitude")
                        cur_fieldmap["type"] = "fieldmap"
                    fieldmap_set.append(cur_fieldmap)
        return fieldmap_set

    def find_match(self, target, source=None):

        # Try to take the easy way out
        if source is not None:
            _target = source.split('.')[0] + '.' + target
            if os.path.exists(_target):
                return target

        if target in list(self.entities.keys()):
            candidates = list(self.entities[target].files.keys())
        else:
            candidates = []

            for root, directories, filenames in os.walk(self.root):
                for f in filenames:
                    if re.search(target + '$', f):
                        if os.path.sep == "\\":
                            f = f.replace("\\", "\\\\")
                        candidates.append(f)

        if source is None:
            return candidates

        # Walk up the file hierarchy from source, find first match
        if not os.path.exists(source):
            raise OSError("The file '%s' doesn't exist." % source)
        elif not source.startswith(self.root):
            raise ValueError("The file '%s' is not contained "
                             "within the current project "
                             "directory (%s)." % (source, self.root))
        rel = os.path.relpath(dirname(source), self.root)
        sep = os.path.sep
        chunks = rel.split(sep)
        n_chunks = len(chunks)
        for i in range(n_chunks, -1, -1):
            path = pathjoin(self.root, *chunks[:i])
            patt = path + '\%s[^\%s]+$' % (sep, sep)
            if sep == "\\":
                patt = path + '\\[^\\]+$'
                patt = patt.replace("\\", "\\\\")
            matches = [x for x in candidates if re.search(patt, x)]
            if matches:
                if len(matches) == 1:
                    return matches[0]
                else:
                    raise ValueError("Ambiguous target: more than one "
                                     "candidate file found in "
                                     "directory '%s'." % path)
        return None
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import print_function
from urllib3 import PoolManager
from pyimgur import Imgur
from requests.exceptions import HTTPError

CLIENT_ID = 'a1dc76d0afc2f34'

def all_casings(input_string):
    """
    Get all case permutations, takes into account non-alphabetic characters.
    Based off of http://stackoverflow.com/a/6792898/2720026
    """
    if not input_string:
        yield ''
    else:
        first = input_string[:1]
        upper, lower = first.upper(), first.lower()
        if lower == upper:
            for sub_casing in all_casings(input_string[1:]):
                yield first + sub_casing
        else:
            for sub_casing in all_casings(input_string[1:]):
                yield lower + sub_casing
                yield upper + sub_casing

def all_casings_sorted(input_string, reverse=True):
    """Convenience function calls sorted on all_casings."""
    return sorted(all_casings(input_string), reverse=reverse)

def exists(urls):
    """Yields each url in urls that exists (returns HTTP status 200)."""
    http = PoolManager(40)
    for url in urls:
        r = http.request('HEAD', url)
        if r.status == 200:
            yield url

def imgur_game_http(s):
    """
    Take string and check for imgur URLs. Must be 5 characters long.
    Returns URLs that exist.
    """
    if len(s) != 5:
        raise RuntimeError('Imgur images are 5 letter long paths. ' +
                           '{} is {} characters long.'.format(s, len(s)))
    permutations = all_casings_sorted(s)
    urls = ('http://imgur.com/'+p for p in permutations)
    return exists(urls)

def imgur_game_api(s):
    """
    Take string and check for imgur URLs. Must be 5 characters long.
    Returns dicts representing URLs that exist. Dict looks like this:
    {
        'url': 'http://imgur.com/daURL',
        'direct': 'http://imgur.com/daURL.png',
        'id': 'daURL'
    }
    """
    if len(s) != 5:
        raise RuntimeError('Imgur images are 5 letter long paths. ' +
                           '{} is {} characters long.'.format(s, len(s)))
    permutations = all_casings_sorted(s)
    im = Imgur(CLIENT_ID)
    existing_urls = exists('http://imgur.com/'+p for p in permutations)
    for i in (id_.replace('http://imgur.com/', '', 1) for id_ in existing_urls):
        try:
            image = im.get_image(i)
            yield {
                'url': 'http://imgur.com/'+i,
                'direct': image.link,
                'id': i,
            }
        except HTTPError:
            pass

def format(it, columns=3):
    """Format iterable into columns. columns=3 by default."""
    c = 1
    for i in it:
        if c < columns:
            print(i, end='  ')
            c += 1
        else:
            c = 1
            print(i)
    if c != 1:
        print()

if __name__ == '__main__':
    import sys

    args = sys.argv[1:]
    for arg in args:
        format(imgur_game_http(arg))
<commit_msg>turns out http HEADing everything is slower than API on Heroku :S<commit_after>#!/usr/bin/env python

from __future__ import print_function
from urllib3 import PoolManager
from pyimgur import Imgur
from requests.exceptions import HTTPError

CLIENT_ID = 'a1dc76d0afc2f34'

def all_casings(input_string):
    """
    Get all case permutations, takes into account non-alphabetic characters.
    Based off of http://stackoverflow.com/a/6792898/2720026
    """
    if not input_string:
        yield ''
    else:
        first = input_string[:1]
        upper, lower = first.upper(), first.lower()
        if lower == upper:
            for sub_casing in all_casings(input_string[1:]):
                yield first + sub_casing
        else:
            for sub_casing in all_casings(input_string[1:]):
                yield lower + sub_casing
                yield upper + sub_casing

def all_casings_sorted(input_string, reverse=True):
    """Convenience function calls sorted on all_casings."""
    return sorted(all_casings(input_string), reverse=reverse)

def exists(urls):
    """Yields each url in urls that exists (returns HTTP status 200)."""
    http = PoolManager(40)
    for url in urls:
        r = http.request('HEAD', url)
        if r.status == 200:
            yield url

def imgur_game_http(s):
    """
    Take string and check for imgur URLs. Must be 5 characters long.
    Returns URLs that exist.
    """
    if len(s) != 5:
        raise RuntimeError('Imgur images are 5 letter long paths. ' +
                           '{} is {} characters long.'.format(s, len(s)))
    permutations = all_casings_sorted(s)
    urls = ('http://imgur.com/'+p for p in permutations)
    return exists(urls)

def imgur_game_api(s):
    """
    Take string and check for imgur URLs. Must be 5 characters long.
    Returns dicts representing URLs that exist. Dict looks like this:
    {
        'url': 'http://imgur.com/daURL',
        'direct': 'http://imgur.com/daURL.png',
        'id': 'daURL'
    }
    """
    if len(s) != 5:
        raise RuntimeError('Imgur images are 5 letter long paths. ' +
                           '{} is {} characters long.'.format(s, len(s)))
    permutations = all_casings_sorted(s)
    im = Imgur(CLIENT_ID)
    for i in permutations:
        try:
            image = im.get_image(i)
            yield {
                'url': 'http://imgur.com/'+i,
                'direct': image.link,
                'id': i,
            }
        except HTTPError:
            pass

def format(it, columns=3):
    """Format iterable into columns. columns=3 by default."""
    c = 1
    for i in it:
        if c < columns:
            print(i, end='  ')
            c += 1
        else:
            c = 1
            print(i)
    if c != 1:
        print()

if __name__ == '__main__':
    import sys

    args = sys.argv[1:]
    for arg in args:
        format(imgur_game_http(arg))
<|endoftext|>