<commit_before>from itertools import chain

from django import forms
from django.template.loader import render_to_string
from django.utils.encoding import force_str
from django.utils.html import conditional_escape
from django.utils.safestring import mark_safe

class SortedCheckboxSelectMultiple(forms.CheckboxSelectMultiple):
    class Media:
        js = (
            'sortedm2m/widget.js',
            'sortedm2m/jquery-ui.js',
        )
        css = {'screen': (
            'sortedm2m/widget.css',
        )}

    def build_attrs(self, attrs=None, **kwargs):  # pylint: disable=arguments-differ
        attrs = dict(attrs or {}, **kwargs)
        attrs = super(SortedCheckboxSelectMultiple, self).build_attrs(attrs)
        classes = attrs.setdefault('class', '').split()
        classes.append('sortedm2m')
        attrs['class'] = ' '.join(classes)
        return attrs

    def render(self, name, value, attrs=None, choices=(), renderer=None):  # pylint: disable=arguments-differ
        if value is None:
            value = []
        has_id = attrs and 'id' in attrs
        final_attrs = self.build_attrs(attrs, name=name)

        # Normalize to strings
        str_values = [force_str(v) for v in value]

        selected = []
        unselected = []

        for i, (option_value, option_label) in enumerate(chain(self.choices, choices)):
            # If an ID attribute was given, add a numeric index as a suffix,
            # so that the checkboxes don't all have the same ID attribute.
            if has_id:
                final_attrs = dict(final_attrs, id='%s_%s' % (attrs['id'], i))
                label_for = ' for="%s"' % conditional_escape(final_attrs['id'])
            else:
                label_for = ''

            cb = forms.CheckboxInput(final_attrs, check_test=lambda value: value in str_values)
            option_value = force_str(option_value)
            rendered_cb = cb.render(name, option_value)
            option_label = conditional_escape(force_str(option_label))
            item = {
                'label_for': label_for,
                'rendered_cb': rendered_cb,
                'option_label': option_label,
                'option_value': option_value
            }
            if option_value in str_values:
                selected.append(item)
            else:
                unselected.append(item)

        # Reorder `selected` array according str_values which is a set of `option_value`s in the
        # order they should be shown on screen
        ordered = []
        for s in str_values:
            for select in selected:
                if s == select['option_value']:
                    ordered.append(select)
        selected = ordered

        html = render_to_string(
            'sortedm2m/sorted_checkbox_select_multiple_widget.html',
            {'selected': selected, 'unselected': unselected})
        return mark_safe(html)

    def value_from_datadict(self, data, files, name):
        value = data.get(name, None)
        if isinstance(value, (str,)):
            return [v for v in value.split(',') if v]
        return value

class SortedMultipleChoiceField(forms.ModelMultipleChoiceField):
    widget = SortedCheckboxSelectMultiple

    def clean(self, value):
        queryset = super(SortedMultipleChoiceField, self).clean(value)
        if value is None or not hasattr(queryset, '__iter__'):
            return queryset
        key = self.to_field_name or 'pk'
        objects = dict((force_str(getattr(o, key)), o) for o in queryset)
        return [objects[force_str(val)] for val in value]

    def has_changed(self, initial, data):
        if initial is None:
            initial = []
        if data is None:
            data = []
        if len(initial) != len(data):
            return True
        initial_set = [force_str(value) for value in self.prepare_value(initial)]
        data_set = [force_str(value) for value in data]
        return data_set != initial_set
<commit_msg>Revert "Remove `admin/js/jquery.init.js` from `SortedCheckboxSelectMultiple`"<commit_after>from itertools import chain

from django import forms
from django.template.loader import render_to_string
from django.utils.encoding import force_str
from django.utils.html import conditional_escape
from django.utils.safestring import mark_safe

class SortedCheckboxSelectMultiple(forms.CheckboxSelectMultiple):
    class Media:
        js = (
            'admin/js/jquery.init.js',
            'sortedm2m/widget.js',
            'sortedm2m/jquery-ui.js',
        )
        css = {'screen': (
            'sortedm2m/widget.css',
        )}

    def build_attrs(self, attrs=None, **kwargs):  # pylint: disable=arguments-differ
        attrs = dict(attrs or {}, **kwargs)
        attrs = super(SortedCheckboxSelectMultiple, self).build_attrs(attrs)
        classes = attrs.setdefault('class', '').split()
        classes.append('sortedm2m')
        attrs['class'] = ' '.join(classes)
        return attrs

    def render(self, name, value, attrs=None, choices=(), renderer=None):  # pylint: disable=arguments-differ
        if value is None:
            value = []
        has_id = attrs and 'id' in attrs
        final_attrs = self.build_attrs(attrs, name=name)

        # Normalize to strings
        str_values = [force_str(v) for v in value]

        selected = []
        unselected = []

        for i, (option_value, option_label) in enumerate(chain(self.choices, choices)):
            # If an ID attribute was given, add a numeric index as a suffix,
            # so that the checkboxes don't all have the same ID attribute.
            if has_id:
                final_attrs = dict(final_attrs, id='%s_%s' % (attrs['id'], i))
                label_for = ' for="%s"' % conditional_escape(final_attrs['id'])
            else:
                label_for = ''

            cb = forms.CheckboxInput(final_attrs, check_test=lambda value: value in str_values)
            option_value = force_str(option_value)
            rendered_cb = cb.render(name, option_value)
            option_label = conditional_escape(force_str(option_label))
            item = {
                'label_for': label_for,
                'rendered_cb': rendered_cb,
                'option_label': option_label,
                'option_value': option_value
            }
            if option_value in str_values:
                selected.append(item)
            else:
                unselected.append(item)

        # Reorder `selected` array according str_values which is a set of `option_value`s in the
        # order they should be shown on screen
        ordered = []
        for s in str_values:
            for select in selected:
                if s == select['option_value']:
                    ordered.append(select)
        selected = ordered

        html = render_to_string(
            'sortedm2m/sorted_checkbox_select_multiple_widget.html',
            {'selected': selected, 'unselected': unselected})
        return mark_safe(html)

    def value_from_datadict(self, data, files, name):
        value = data.get(name, None)
        if isinstance(value, (str,)):
            return [v for v in value.split(',') if v]
        return value

class SortedMultipleChoiceField(forms.ModelMultipleChoiceField):
    widget = SortedCheckboxSelectMultiple

    def clean(self, value):
        queryset = super(SortedMultipleChoiceField, self).clean(value)
        if value is None or not hasattr(queryset, '__iter__'):
            return queryset
        key = self.to_field_name or 'pk'
        objects = dict((force_str(getattr(o, key)), o) for o in queryset)
        return [objects[force_str(val)] for val in value]

    def has_changed(self, initial, data):
        if initial is None:
            initial = []
        if data is None:
            data = []
        if len(initial) != len(data):
            return True
        initial_set = [force_str(value) for value in self.prepare_value(initial)]
        data_set = [force_str(value) for value in data]
        return data_set != initial_set
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Wed Jul 15 13:39:36 2020

@author: mario
"""

from github import Github
import os

file = open("../../credits.md")
for line in file:
    print(line)

g = Github( os.getenv("GITHUB_TOKEN"))
# Github Enterprise with custom hostname

# Then play with your Github objects:
repo = g.get_repo(os.getenv("GITHUB_REPO"))

contents = repo.get_contents("")
while contents:
    file_content = contents.pop(0)
    if file_content.type == "dir":
        contents.extend(repo.get_contents(file_content.path))
    else:
        print(file_content)
        if file_content[-3:] == ".md":
            print("selected: "+ file_content)

print("finished")<commit_msg>Update fuzz.py<commit_after># -*- coding: utf-8 -*-
"""
Created on Wed Jul 15 13:39:36 2020

@author: mario
"""

from github import Github
import os

file = open("../../credits.md")
for line in file:
    print(line)

g = Github( os.getenv("GITHUB_TOKEN"))
# Github Enterprise with custom hostname

# Then play with your Github objects:
repo = g.get_repo(os.getenv("GITHUB_REPO"))

contents = repo.get_contents("")
while contents:
    file_content = contents.pop(0)
    if file_content.type == "dir":
        contents.extend(repo.get_contents(file_content.path))
    else:
        print(file_content.url)
        if file_content.path[-3:] == ".md":
            print("selected: "+ file_content)

print("finished")<|endoftext|>


<commit_before><commit_msg>Dynamically set the directory to upload to<commit_after><|endoftext|>


<commit_before>
# -*- encoding: utf-8 -*-
##############################################################################
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see http://www.gnu.org/licenses/.
#
##############################################################################

from openerp.osv import orm, fields
from datetime import datetime
from dateutil.relativedelta import relativedelta

class StockProductionLot(orm.Model):
    _inherit = "stock.production.lot"
    _columns = {
        "warrant_limit": fields.datetime("Warranty")}

    def create(self, cr, uid, vals, context=None):
        if context is None:
            context = {}
        if vals.get('product_id'):
            product = self.pool["product.product"].browse(
                cr, uid, vals["product_id"], context=context)
            create_date = ('create_date' in vals and
                           vals['create_date'] or datetime.now())
            if 'sup_warrant' not in context:
                warrant_limit = (
                    product.warranty and
                    (create_date +
                     relativedelta(months=int(product.warranty))))
            else:
                warrant_limit = (
                    create_date +
                    relativedelta(months=int(context['sup_warrant']) +
                                  relativedelta(
                        days=int(31 * (context['sup_warrant'] -
                                       int(context['sup_warrant']))))))
            if warrant_limit:
                vals.update({'warrant_limit': warrant_limit})
        return super(StockProductionLot, self).create(cr, uid, vals,
                                                      context=context)

class StockPackOperation(orm.Model):

    _inherit = "stock.pack.operation"

    def create_and_assign_lot(self, cr, uid, id, name, context=None):
        if context is None:
            context = {}
        pack_oper = self.browse(cr, uid, id, context=context)
        if pack_oper.picking_id.location_id.usage == 'supplier':
            sup_obj = self.pool['product.supplierinfo']
            suppinfo_id = sup_obj.search(
                cr, uid, [('name', '=', pack_oper.picking_id.partner_id.id),
                          ('product_tmpl_id', '=',
                           pack_oper.product_id.product_tmpl_id.id)],
                context=context)
            sup = sup_obj.browse(cr, uid, suppinfo_id, context=context)
            sup_warrant = sup and sup[0].warrant_months
            ctx = context.copy()
            ctx.update({'sup_warrant': sup_warrant})
        return super(StockPackOperation, self).create_and_assign_lot(
            cr, uid, id, name, context=ctx)
<commit_msg>[mod] remove float calculations<commit_after>
# -*- encoding: utf-8 -*-
##############################################################################
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see http://www.gnu.org/licenses/.
#
##############################################################################

from openerp.osv import orm, fields
from datetime import datetime
from dateutil.relativedelta import relativedelta

class StockProductionLot(orm.Model):
    _inherit = "stock.production.lot"
    _columns = {
        "warrant_limit": fields.datetime("Warranty")}

    def create(self, cr, uid, vals, context=None):
        if context is None:
            context = {}
        if vals.get('product_id'):
            product = self.pool["product.product"].browse(
                cr, uid, vals["product_id"], context=context)
            create_date = ('create_date' in vals and
                           vals['create_date'] or datetime.now())
            if 'sup_warrant' not in context:
                warrant_limit = (
                    product.warranty and
                    (create_date +
                     relativedelta(months=int(product.warranty))))
            else:
                warrant_limit = (
                    create_date +
                    relativedelta(months=context['sup_warrant']))
            if warrant_limit:
                vals.update({'warrant_limit': warrant_limit})
        return super(StockProductionLot, self).create(cr, uid, vals,
                                                      context=context)

class StockPackOperation(orm.Model):

    _inherit = "stock.pack.operation"

    def create_and_assign_lot(self, cr, uid, id, name, context=None):
        if context is None:
            context = {}
        ctx = context.copy()
        pack_oper = self.browse(cr, uid, id, context=context)
        if pack_oper.picking_id.location_id.usage == 'supplier':
            sup_obj = self.pool['product.supplierinfo']
            suppinfo_id = sup_obj.search(
                cr, uid, [('name', '=', pack_oper.picking_id.partner_id.id),
                          ('product_tmpl_id', '=',
                           pack_oper.product_id.product_tmpl_id.id)],
                context=context)
            sup = sup_obj.browse(cr, uid, suppinfo_id, context=context)
            sup_warrant = sup and sup[0].warrant_months
            ctx.update({'sup_warrant': sup_warrant})
        return super(StockPackOperation, self).create_and_assign_lot(
            cr, uid, id, name, context=ctx)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Nikita Kouevda
# 2014/12/14

from argparse import ArgumentParser
from html.parser import HTMLParser
import multiprocessing
import os
import re
import sys
from urllib.request import urlopen

def print_error(message):
  print('{}: {}'.format(sys.argv[0], message), file=sys.stderr)

def save_image(url, path, verbose=False):
  if os.path.exists(path):
    print_error('file exists: {}'.format(path))
    return

  try:
    with urlopen(url) as in_file:
      content = in_file.read()
    if verbose:
      print('saving file: {}'.format(path))
    with open(path, 'wb') as out_file:
      out_file.write(content)
  except:
    print_error('could not save file: {}'.format(path))

def save_albums(albums, numbers=False, titles=False, verbose=False):
  html_parser = HTMLParser()

  re_album_hash = re.compile(r'^(?:.*/)?([A-Za-z0-9]{5})(?:[/?#].*)?$')
  re_image = re.compile(r'"hash":"([A-Za-z0-9]{5,7})".+?"ext":"(.+?)"')
  re_title = re.compile(r'data-title="(.*?)"')
  re_title_sanitize = re.compile(r'(?:[^ -~]|[/:])+')

  pool = multiprocessing.Pool()

  for album in albums:
    try:
      album_hash = re_album_hash.search(album).group(1)
      with urlopen('http://imgur.com/a/{}'.format(album_hash)) as in_file:
        content = in_file.read().decode()
    except:
      print_error('could not read album: {}'.format(album))
      continue

    if titles:
      title_raw = re_title.search(content).group(1)
      title_unescaped = html_parser.unescape(title_raw)
      title_sanitized = re_title_sanitize.sub(' ', title_unescaped)
      album_dir = '{}-[{}]'.format(album_hash, title_sanitized)
    else:
      album_dir = album_hash

    if not os.path.exists(album_dir):
      if verbose:
        print('making directory: {}'.format(album_dir))
      os.makedirs(album_dir)

    for i, image_match in enumerate(re_image.finditer(content)):
      orig_name = '{}{}'.format(image_match.group(1), image_match.group(2))
      url = 'http://i.imgur.com/{}'.format(orig_name)
      filename = '{:d}-{}'.format(i, orig_name) if numbers else orig_name
      path = '{}/{}'.format(album_dir, filename)
      pool.apply_async(save_image, args=(url, path, verbose))

  pool.close()
  pool.join()

def main():
  parser = ArgumentParser(description='Imgur album downloader')
  parser.add_argument('albums', nargs='+', type=str, metavar='album',
                      help='an album hash or URL')
  parser.add_argument('-n', '--numbers', action='store_true',
                      help='prepend numbers to filenames')
  parser.add_argument('-t', '--titles', action='store_true',
                      help='append album titles to directory names')
  parser.add_argument('-v', '--verbose', action='store_true',
                      help='verbose output')

  args = parser.parse_args()
  save_albums(args.albums, numbers=args.numbers, titles=args.titles,
              verbose=args.verbose)

if __name__ == '__main__':
  main()
<commit_msg>Use logging module instead of print to stderr<commit_after>#!/usr/bin/env python3

# Nikita Kouevda
# 2015/05/23

from argparse import ArgumentParser
from html.parser import HTMLParser
import logging
import multiprocessing
import os
import re
from urllib.request import urlopen

def save_image(url, path):
  if os.path.exists(path):
    logging.error('file exists: %s', path)
    return

  try:
    with urlopen(url) as in_file:
      content = in_file.read()
    logging.info('saving file: %s', path)
    with open(path, 'wb') as out_file:
      out_file.write(content)
  except:
    logging.exception('could not save file: %s', path)

def save_albums(albums, numbers=False, titles=False):
  html_parser = HTMLParser()

  re_album_hash = re.compile(r'^(?:.*/)?([A-Za-z0-9]{5})(?:[/?#].*)?$')
  re_image = re.compile(r'"hash":"([A-Za-z0-9]{5,7})".+?"ext":"(.+?)"')
  re_title = re.compile(r'data-title="(.*?)"')
  re_title_sanitize = re.compile(r'(?:[^ -~]|[/:])+')

  pool = multiprocessing.Pool()

  for album in albums:
    try:
      album_hash = re_album_hash.search(album).group(1)
      with urlopen('http://imgur.com/a/{}'.format(album_hash)) as in_file:
        content = in_file.read().decode()
    except:
      logging.exception('could not read album: %s', album)
      continue

    if titles:
      title_raw = re_title.search(content).group(1)
      title_unescaped = html_parser.unescape(title_raw)
      title_sanitized = re_title_sanitize.sub(' ', title_unescaped)
      album_dir = '{}-[{}]'.format(album_hash, title_sanitized)
    else:
      album_dir = album_hash

    if not os.path.exists(album_dir):
      logging.info('making directory: %s', album_dir)
      os.makedirs(album_dir)

    for i, image_match in enumerate(re_image.finditer(content)):
      orig_name = '{}{}'.format(image_match.group(1), image_match.group(2))
      url = 'http://i.imgur.com/{}'.format(orig_name)
      filename = '{:d}-{}'.format(i, orig_name) if numbers else orig_name
      path = '{}/{}'.format(album_dir, filename)
      pool.apply_async(save_image, args=(url, path))

  pool.close()
  pool.join()

def main():
  parser = ArgumentParser(description='Imgur album downloader')
  parser.add_argument('albums', nargs='+', type=str, metavar='album',
                      help='an album hash or URL')
  parser.add_argument('-n', '--numbers', action='store_true',
                      help='prepend numbers to filenames')
  parser.add_argument('-t', '--titles', action='store_true',
                      help='append album titles to directory names')
  parser.add_argument('-v', '--verbose', action='store_true',
                      help='verbose output')
  args = parser.parse_args()

  logging.basicConfig(format='%(levelname)s: %(message)s',
                      level=logging.INFO if args.verbose else logging.WARNING)

  save_albums(args.albums, numbers=args.numbers, titles=args.titles)

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3.5
import re
from datetime import datetime
from rdflib.plugins.serializers.turtle import TurtleSerializer
from rdflib import RDF, RDFS, OWL, BNode, URIRef
from rdflib.namespace import SKOS, DC, Namespace

OBOANN = Namespace('http://ontology.neuinfo.org/NIF/Backend/OBO_annotation_properties.owl#')
BIRNANN = Namespace('http://ontology.neuinfo.org/NIF/Backend/BIRNLex_annotation_properties.owl#')
oboInOwl = Namespace('http://www.geneontology.org/formats/oboInOwl#')
#IAO = Namespace('http://purl.obolibrary.org/obo/IAO_')  # won't work because numbers ...

def natsort(s, pat=re.compile(r'([0-9]+)')):
    return [int(t) if t.isdigit() else t.lower() for t in pat.split(s)]

# XXX WARNING prefixes are not 100% deterministic if there is more than one prefix for namespace
#     the implementation of IOMemory.bind in rdflib means that the last prefix defined in the list
#     will likely be the one that is called when NamespaceManager.compute_qname calls self.store.prefix

# desired behavior (XXX does not match the implementation!
# 1) if there is more than one entry at a level URIRef goes first natsorted then lists then predicate lists then subject lists
# 2) sorting for nested structures in a list determined by
#     a) rank of object attached to the highest ranked predicate (could just be alpha)
#     b) rank of object attached to the second highest ranked predicate
#     c) where there are multiple of the same predicate their own rank is determined by the ranks of their objects
#     d) predicate lists are ranked -2 and subject (proper?) lists are ranked -1 as predicates
#     e) sorting proper lists... get the predicate rank and then the rank of the value
# object type ranks:
#  1 URIRef
#  2 predicate list []
#  3 proper list ()
# object value ranks:
#  1 URIRef -> alphabetical
#  2 lists -> object type ranks of their nth elements
#
# a nice example is NIF-Cell:nlx_cell_091210 in NIF-Neuron-BrainRegion-Bridge.ttl

SUBJECT = 0
VERB = 1
OBJECT = 2

def qname_mp(self, uri):  # for monkey patching Graph
    try:
        prefix, namespace, name = self.compute_qname(uri, False)
    except Exception:  # no prefix no problems
        return uri

    if prefix == "":
        return name
    else:
        return ":".join((prefix, name))

class CustomTurtleSerializer(TurtleSerializer):
    """ NIFSTD custom ttl serliziation """

    topClasses = [RDFS.Class,
                  OWL.Ontology,
                  OWL.ObjectProperty,
                  RDFS.Datatype,
                  OWL.AnnotationProperty,
                  OWL.DatatypeProperty,
                  OWL.Class,
                  OWL.NamedIndividual,
                  OWL.AllDifferent,
                 ]

    SECTIONS = ('',
                '',
                '\n### Object Properties\n',
                '\n### Datatypes\n',
                '\n### Annotation Properties\n',
                '\n### Data Properties\n',
                '\n### Classes\n',
                '\n### Individuals\n',
                '\n### Axioms\n',
                '\n### Annotations\n',
               )

    predicateOrder = [RDF.type,
                      OWL.onProperty,
                      OWL.allValuesFrom,
                      OWL.someValuesFrom,
                      OWL.imports,
                      OWL.deprecated,
                      URIRef('http://purl.obolibrary.org/obo/IAO_0100001'),  # replacedBy:
                      oboInOwl.hasDbXref,
                      OWL.equivalentClass,
                      RDFS.label,
                      SKOS.prefLabel,
                      OBOANN.synonym,
                      OBOANN.abbrev,
                      DC.title,
                      SKOS.definition,
                      DC.description,
                      RDFS.subClassOf,
                      OWL.intersectionOf,
                      OWL.unionOf,
                      OWL.disjointWith,
                      OWL.disjointUnionOf,
                      RDFS.comment,
                      SKOS.note,
                      SKOS.editorialNote,
                      SKOS.changeNote,
                      OWL.versionInfo,
                      OBOANN.createdDate,
                      OBOANN.modifiedDate,
                     ]

    def __init__(self, store):
        setattr(store.__class__, 'qname', qname_mp)  # monkey patch to fix generate=True
        store.namespace_manager.reset()  # ensure that the namespace_manager cache doesn't lead to non deterministic ser
        super(CustomTurtleSerializer, self).__init__(store)
        self.object_rank = {o:i  # global rank for all URIRef that appear as objects
                            for i, o in
                            enumerate(
                                sorted(sorted(set([_ for _ in self.store.objects(None, None)
                                            if not isinstance(_, BNode)] +  # URIRef + Literal
                                           [_ for _ in self.store.subjects(None, None)
                                            if isinstance(_, URIRef)])),
                                       key=lambda _: natsort(self.store.qname(_))))}

        self.node_rank = {}
        def recurse(node, rank):  # XXX warning: cycles?
            for s in self.store.subjects(None, node):
                if isinstance(s, BNode):  # w/o this we break recursion limit
                    if s not in self.node_rank:
                        self.node_rank[s] = rank
                    else:
                        self.node_rank[s] += rank
                    recurse(s, rank)  # if we are retracing steps we already added previous ranks to upstream so don't need to propagate again

        for o, r in self.object_rank.items():
            recurse(o, r)

    def _globalSortKey(self, bnode):
        if isinstance(bnode, BNode):
            try:
                return self.node_rank[bnode]
            except KeyError as e :
                # This is what we have to contend with here :/
                # ro:proper_part_of oboInOwl:hasDefinition [ oboInOwl:hasDbXref [ ] ] .
                print('WARNING: some node that is an object isnt really an object?')
                print(e)
                return -1
        else:  # every Literal and URIRef object has a global rank now
            return self.object_rank[bnode]

    def startDocument(self):  # modified to natural sort prefixes
        self._started = True
        ns_list = sorted(sorted(self.namespaces.items()), key=lambda kv: (natsort(kv[0]), kv[1]))
        for prefix, uri in ns_list:
            self.write(self.indent() + '@prefix %s: <%s> .\n' % (prefix, uri))
        if ns_list and self._spacious:
            self.write('\n')

    def orderSubjects(self):  # modified to enable natural sort of subjects
        seen = {}
        sections = []

        def key(m):
            if not isinstance(m, BNode):
                m = self.store.qname(m)
            return natsort(m)

        for i, classURI in enumerate(self.topClasses):  # SECTIONS
            members = sorted(self.store.subjects(RDF.type, classURI))
            members.sort(key=key)

            subjects = []
            for member in members:
                subjects.append(member)
                self._topLevels[member] = True
                seen[member] = True
            sections.append(subjects)

        recursable = [
            (isinstance(subject, BNode),
             self._references[subject], subject)
            for subject in self._subjects if subject not in seen]

        recursable.sort(key=lambda t: self._globalSortKey(t[-1]))

        sections[-1].extend([subject for (isbnode, refs, subject) in recursable if isbnode])  # group bnodes with classes
        sections.append([subject for (isbnode, refs, subject) in recursable if not isbnode])  # annotation targets

        return sections

    def _predicateList(self, subject, newline=False):  # XXX unmodified
        properties = self.buildPredicateHash(subject)
        propList = self.sortProperties(properties)
        if len(propList) == 0:
            return
        self.verb(propList[0], newline=newline)
        self.objectList(properties[propList[0]])
        for predicate in propList[1:]:
            self.write(' ;\n' + self.indent(1))
            self.verb(predicate, newline=True)
            self.objectList(properties[predicate])

    def sortProperties(self, properties):  # modified to sort objects using their global rank
        """Take a hash from predicate uris to lists of values.
           Sort the lists of values.  Return a sorted list of properties."""
        # Sort object lists
        for prop, objects in properties.items():
            objects.sort(key=self._globalSortKey)

        # Make sorted list of properties
        propList = []
        seen = {}
        for prop in self.predicateOrder:
            if (prop in properties) and (prop not in seen):
                propList.append(prop)
                seen[prop] = True
        props = sorted(properties.keys())
        props.sort(key=natsort)
        for prop in props:
            if prop not in seen:
                propList.append(prop)
                seen[prop] = True
        return propList

    def _buildPredicateHash(self, subject):  # XXX unmodified
        """
        Build a hash key by predicate to a list of objects for the given
        subject
        """
        properties = {}
        for s, p, o in self.store.triples((subject, None, None)):
            oList = properties.get(p, [])
            oList.append(o)
            properties[p] = oList

        return properties

    def doList(self, l):  # modified to put rdf list items on new lines
        while l:
            item = self.store.value(l, RDF.first)
            if item is not None:
                self.write('\n' + self.indent(1))
                self.path(item, OBJECT, newline=True)
                self.subjectDone(l)
            l = self.store.value(l, RDF.rest)

    def _p_default(self, node, position, newline=False):  # XXX unmodified
        if position != SUBJECT and not newline:
            self.write(' ')
        self.write(self.label(node, position))
        return True

    def _p_squared(self, node, position, newline=False):  # XXX unmodified
        if (not isinstance(node, BNode)
                or node in self._serialized
                or self._references[node] > 1
                or position == SUBJECT):
            return False

        if not newline:
            self.write(' ')

        if self.isValidList(node):
            # this is a list
            self.write('(')
            self.depth += 1  # 2
            self.doList(node)
            self.depth -= 1  # 2
            self.write(' )')
        else:
            self.subjectDone(node)
            self.depth += 2
            # self.write('[\n' + self.indent())
            self.write('[')
            self.depth -= 1
            # self.predicateList(node, newline=True)
            self.predicateList(node, newline=False)
            # self.write('\n' + self.indent() + ']')
            self.write(' ]')
            self.depth -= 1

        return True

    def _s_default(self, subject):  # XXX unmodified, ordering issues start here
        self.write('\n' + self.indent())
        self.path(subject, SUBJECT)
        self.predicateList(subject)
        self.write(' .')
        return True

    def _s_squared(self, subject):  # XXX unmodified, ordering issues start here
        if (self._references[subject] > 0) or not isinstance(subject, BNode):
            return False
        self.write('\n' + self.indent() + '[]')
        self.predicateList(subject)
        self.write(' .')
        return True

    def serialize(self, stream, base=None, encoding=None,  # modified to enable section headers
                  spacious=None, **args):
        self.reset()
        self.stream = stream
        self.base = base

        if spacious is not None:
            self._spacious = spacious

        self.preprocess()
        sections_list = self.orderSubjects()

        self.startDocument()

        firstTime = True
        for header, subjects_list in zip(self.SECTIONS, sections_list):
            if header == self.SECTIONS[-2]:  # Axioms need to not be bnodes...
                if any([type(s) != BNode for s in subjects_list]):
                    print('GOT ONE')
                    self.write(header)
            elif subjects_list:
                self.write(header)
            for subject in subjects_list:
                if self.isDone(subject):
                    continue
                if firstTime:
                    firstTime = False
                if self.statement(subject) and not firstTime:
                    self.write('\n')

        self.endDocument()
        stream.write(u"\n".encode('ascii'))
        stream.write((u"### Serialized using the nifstd custom serializer v1.0.2\n").encode('ascii'))

<commit_msg>ttlser work on propagating object_rank, not finished<commit_after>#!/usr/bin/env python3.5
import re
from datetime import datetime
from rdflib.plugins.serializers.turtle import TurtleSerializer
from rdflib import RDF, RDFS, OWL, BNode, URIRef
from rdflib.namespace import SKOS, DC, Namespace
from desc.util.ipython import embed

OBOANN = Namespace('http://ontology.neuinfo.org/NIF/Backend/OBO_annotation_properties.owl#')
BIRNANN = Namespace('http://ontology.neuinfo.org/NIF/Backend/BIRNLex_annotation_properties.owl#')
oboInOwl = Namespace('http://www.geneontology.org/formats/oboInOwl#')
#IAO = Namespace('http://purl.obolibrary.org/obo/IAO_')  # won't work because numbers ...

def natsort(s, pat=re.compile(r'([0-9]+)')):
    return [int(t) if t.isdigit() else t.lower() for t in pat.split(s)]

# XXX WARNING prefixes are not 100% deterministic if there is more than one prefix for namespace
#     the implementation of IOMemory.bind in rdflib means that the last prefix defined in the list
#     will likely be the one that is called when NamespaceManager.compute_qname calls self.store.prefix

# desired behavior (XXX does not match the implementation!
# 1) if there is more than one entry at a level URIRef goes first natsorted then lists then predicate lists then subject lists
# 2) sorting for nested structures in a list determined by
#     a) rank of object attached to the highest ranked predicate (could just be alpha)
#     b) rank of object attached to the second highest ranked predicate
#     c) where there are multiple of the same predicate their own rank is determined by the ranks of their objects
#     d) predicate lists are ranked -2 and subject (proper?) lists are ranked -1 as predicates
#     e) sorting proper lists... get the predicate rank and then the rank of the value
# object type ranks:
#  1 URIRef
#  2 predicate list []
#  3 proper list ()
# object value ranks:
#  1 URIRef -> alphabetical
#  2 lists -> object type ranks of their nth elements
#
# a nice example is NIF-Cell:nlx_cell_091210 in NIF-Neuron-BrainRegion-Bridge.ttl

SUBJECT = 0
VERB = 1
OBJECT = 2

def qname_mp(self, uri):  # for monkey patching Graph
    try:
        prefix, namespace, name = self.compute_qname(uri, False)
    except Exception:  # no prefix no problems
        return uri

    if prefix == "":
        return name
    else:
        return ":".join((prefix, name))

class CustomTurtleSerializer(TurtleSerializer):
    """ NIFSTD custom ttl serliziation """

    topClasses = [RDFS.Class,
                  OWL.Ontology,
                  OWL.ObjectProperty,
                  RDFS.Datatype,
                  OWL.AnnotationProperty,
                  OWL.DatatypeProperty,
                  OWL.Class,
                  OWL.NamedIndividual,
                  OWL.AllDifferent,
                 ]

    SECTIONS = ('',
                '',
                '\n### Object Properties\n',
                '\n### Datatypes\n',
                '\n### Annotation Properties\n',
                '\n### Data Properties\n',
                '\n### Classes\n',
                '\n### Individuals\n',
                '\n### Axioms\n',
                '\n### Annotations\n',
               )

    predicateOrder = [RDF.type,
                      OWL.onProperty,
                      OWL.allValuesFrom,
                      OWL.someValuesFrom,
                      OWL.imports,
                      OWL.deprecated,
                      URIRef('http://purl.obolibrary.org/obo/IAO_0100001'),  # replacedBy:
                      oboInOwl.hasDbXref,
                      OWL.equivalentClass,
                      RDFS.label,
                      SKOS.prefLabel,
                      OBOANN.synonym,
                      OBOANN.abbrev,
                      DC.title,
                      SKOS.definition,
                      DC.description,
                      RDFS.subClassOf,
                      OWL.intersectionOf,
                      OWL.unionOf,
                      OWL.disjointWith,
                      OWL.disjointUnionOf,
                      RDFS.comment,
                      SKOS.note,
                      SKOS.editorialNote,
                      SKOS.changeNote,
                      OWL.versionInfo,
                      OBOANN.createdDate,
                      OBOANN.modifiedDate,
                     ]

    def __init__(self, store):
        setattr(store.__class__, 'qname', qname_mp)  # monkey patch to fix generate=True
        store.namespace_manager.reset()  # ensure that the namespace_manager cache doesn't lead to non deterministic ser
        super(CustomTurtleSerializer, self).__init__(store)
        self.object_rank = {o:i  # global rank for all URIRef that appear as objects
                            for i, o in
                            enumerate(
                                sorted(
                                    sorted(set(
                                        [_ for _ in self.store.objects(None, None)
                                         if not isinstance(_, BNode)] +  # URIRef + Literal
                                        [_ for _ in self.store.subjects(None, None)
                                         if isinstance(_, URIRef)])),
                                    key=lambda _: natsort(self.store.qname(_))))}

        self.node_rank = {}
        def recurse(node, rank):  # XXX warning: cycles?
            eranks = 0
            for s in self.store.subjects(None, node):
                if isinstance(s, BNode):  # w/o this we break recursion limit
                    if s not in self.node_rank:
                        self.node_rank[s] = rank
                    else:
                        self.node_rank[s] += rank
                    eranks += recurse(s, rank)  # only propagate if erank comes from object_rank
                elif isinstance(node, BNode):  # this is the terminal case
                    eranks += self.object_rank[s]
                    #print('incremented rank of', node, 'by rank for', s, 'which is', erank)
            if isinstance(node, BNode):  # this is the terminal case
                self.node_rank[node] += eranks
            return eranks

        for o, r in self.object_rank.items():
            recurse(o, r)

    def _globalSortKey(self, bnode):
        if isinstance(bnode, BNode):
            try:
                return self.node_rank[bnode]
            except KeyError as e :
                # This is what we have to contend with here :/
                # ro:proper_part_of oboInOwl:hasDefinition [ oboInOwl:hasDbXref [ ] ] .
                print('WARNING: some node that is an object isnt really an object?')
                print(e)
                return -1
        else:  # every Literal and URIRef object has a global rank now
            return self.object_rank[bnode]

    def startDocument(self):  # modified to natural sort prefixes
        self._started = True
        ns_list = sorted(sorted(self.namespaces.items()), key=lambda kv: (natsort(kv[0]), kv[1]))
        for prefix, uri in ns_list:
            self.write(self.indent() + '@prefix %s: <%s> .\n' % (prefix, uri))
        if ns_list and self._spacious:
            self.write('\n')

    def orderSubjects(self):  # modified to enable natural sort of subjects
        seen = {}
        sections = []

        def key(m):
            if not isinstance(m, BNode):
                m = self.store.qname(m)
            return natsort(m)

        for i, classURI in enumerate(self.topClasses):  # SECTIONS
            members = sorted(self.store.subjects(RDF.type, classURI))
            members.sort(key=key)

            subjects = []
            for member in members:
                subjects.append(member)
                self._topLevels[member] = True
                seen[member] = True
            sections.append(subjects)

        recursable = [
            (isinstance(subject, BNode),
             self._references[subject], subject)
            for subject in self._subjects if subject not in seen]

        recursable.sort(key=lambda t: self._globalSortKey(t[-1]))

        sections[-1].extend([subject for (isbnode, refs, subject) in recursable if isbnode])  # group bnodes with classes
        sections.append([subject for (isbnode, refs, subject) in recursable if not isbnode])  # annotation targets

        return sections

    def _predicateList(self, subject, newline=False):  # XXX unmodified
        properties = self.buildPredicateHash(subject)
        propList = self.sortProperties(properties)
        if len(propList) == 0:
            return
        self.verb(propList[0], newline=newline)
        self.objectList(properties[propList[0]])
        for predicate in propList[1:]:
            self.write(' ;\n' + self.indent(1))
            self.verb(predicate, newline=True)
            self.objectList(properties[predicate])

    def sortProperties(self, properties):  # modified to sort objects using their global rank
        """Take a hash from predicate uris to lists of values.
           Sort the lists of values.  Return a sorted list of properties."""
        # Sort object lists
        for prop, objects in properties.items():
            objects.sort(key=self._globalSortKey)

        # Make sorted list of properties
        propList = []
        seen = {}
        for prop in self.predicateOrder:
            if (prop in properties) and (prop not in seen):
                propList.append(prop)
                seen[prop] = True
        props = sorted(properties.keys())
        props.sort(key=natsort)
        for prop in props:
            if prop not in seen:
                propList.append(prop)
                seen[prop] = True
        return propList

    def _buildPredicateHash(self, subject):  # XXX unmodified
        """
        Build a hash key by predicate to a list of objects for the given
        subject
        """
        properties = {}
        for s, p, o in self.store.triples((subject, None, None)):
            oList = properties.get(p, [])
            oList.append(o)
            properties[p] = oList

        return properties

    def doList(self, l):  # modified to put rdf list items on new lines
        while l:
            item = self.store.value(l, RDF.first)
            if item is not None:
                self.write('\n' + self.indent(1))
                self.path(item, OBJECT, newline=True)
                self.subjectDone(l)
            l = self.store.value(l, RDF.rest)

    def _p_default(self, node, position, newline=False):  # XXX unmodified
        if position != SUBJECT and not newline:
            self.write(' ')
        self.write(self.label(node, position))
        return True

    def _p_squared(self, node, position, newline=False):  # XXX unmodified
        if (not isinstance(node, BNode)
                or node in self._serialized
                or self._references[node] > 1
                or position == SUBJECT):
            return False

        if not newline:
            self.write(' ')

        if self.isValidList(node):
            # this is a list
            self.write('(')
            self.depth += 1  # 2
            self.doList(node)
            self.depth -= 1  # 2
            self.write(' )')
        else:
            self.subjectDone(node)
            self.depth += 2
            # self.write('[\n' + self.indent())
            self.write('[')
            self.depth -= 1
            # self.predicateList(node, newline=True)
            self.predicateList(node, newline=False)
            # self.write('\n' + self.indent() + ']')
            self.write(' ]')
            self.depth -= 1

        return True

    def _s_default(self, subject):  # XXX unmodified, ordering issues start here
        self.write('\n' + self.indent())
        self.path(subject, SUBJECT)
        self.predicateList(subject)
        self.write(' .')
        return True

    def _s_squared(self, subject):  # XXX unmodified, ordering issues start here
        if (self._references[subject] > 0) or not isinstance(subject, BNode):
            return False
        self.write('\n' + self.indent() + '[]')
        self.predicateList(subject)
        self.write(' .')
        return True

    def serialize(self, stream, base=None, encoding=None,  # modified to enable section headers
                  spacious=None, **args):
        self.reset()
        self.stream = stream
        self.base = base

        if spacious is not None:
            self._spacious = spacious

        self.preprocess()
        sections_list = self.orderSubjects()

        self.startDocument()

        firstTime = True
        for header, subjects_list in zip(self.SECTIONS, sections_list):
            if header == self.SECTIONS[-2]:  # Axioms need to not be bnodes...
                if any([type(s) != BNode for s in subjects_list]):
                    print('GOT ONE')
                    self.write(header)
            elif subjects_list:
                self.write(header)
            for subject in subjects_list:
                if self.isDone(subject):
                    continue
                if firstTime:
                    firstTime = False
                if self.statement(subject) and not firstTime:
                    self.write('\n')

        self.endDocument()
        stream.write(u"\n".encode('ascii'))
        stream.write((u"### Serialized using the nifstd custom serializer v1.0.2\n").encode('ascii'))

<|endoftext|>


<commit_before>#
# Copyright (c) 2008 rPath, Inc.
#
from lxml import etree

from catalogService.rest.response import XmlStringResponse
from catalogService import http_codes

class CatalogErrorResponse(XmlStringResponse):
    def __init__(self, status, message, tracebackData='', productCodeData=None, envelopeStatus=None,
                 *args, **kw):
        # See RBL-3818 - flex does not expose the content of a non-200
        # response, so we have to tunnel faults through 200.
        faultNode = etree.Element("fault")
        node = etree.Element("code")
        node.text = str(status)
        faultNode.append(node)

        node = etree.Element("message")
        node.text = message
        faultNode.append(node)

        if tracebackData:
            node = etree.Element("traceback")
            node.text = tracebackData
            faultNode.append(node)

        if productCodeData:
            node = etree.Element("productCode")
            for k, v in productCodeData.iteritems():
                node.set(k, v)
            faultNode.append(node)

        content = etree.tostring(faultNode, pretty_print = True,
            xml_declaration = True, encoding = 'UTF-8')
        # Prefer envelopeStatus if set, otherwise use status
        XmlStringResponse.__init__(self, content=content,
                                   status=envelopeStatus or status,
                                   message=message, *args, **kw)

class CatalogError(Exception):
    """Base class for errors from Cloud Catalog Service"""
    status = http_codes.HTTP_INTERNAL_SERVER_ERROR
    def __init__(self, message=None, status=None, *args, **kw):
        if not message:
            message = self.__class__.__doc__
        if not status:
            status = self.status
        self.status = status
        self.message = message
        self.tracebackData = kw.get('tracebackData', None)
        self.productCodeData = kw.get('productCodeData', None)

    def __str__(self):
        return self.message

class InvalidCloudName(CatalogError):
    """Cloud name is not valid"""
    status = http_codes.HTTP_NOT_FOUND

class MissingCredentials(CatalogError):
    """Cloud credentials are not set in rBuilder"""
    status = http_codes.HTTP_BAD_REQUEST

class PermissionDenied(CatalogError):
    """Permission Denied"""
    status = http_codes.HTTP_FORBIDDEN

class ParameterError(CatalogError):
    """Errors were detected in input"""
    status = http_codes.HTTP_BAD_REQUEST
    def __init__(self, message = None):
        CatalogError.__init__(self, message = message)

class ResponseError(CatalogError):
    """Response error from remote cloud service"""
    status = http_codes.HTTP_BAD_REQUEST
    # XXX flex's httpd stack requires we pass a 200 or it will junk the content
    def __init__(self, status, message, body, productCodeData=None):
        # strip any xml tags
        if body.strip().startswith('<?xml'):
            body = ''.join(body.splitlines(True)[1:])
        CatalogError.__init__(self, message, status = status, tracebackData = body, productCodeData = productCodeData)

class CloudExists(CatalogError):
    """Target already exists"""
    status = http_codes.HTTP_CONFLICT

class HttpNotFound(CatalogError):
    """File not found"""
    status = 404

class ErrorMessageCallback(object):
    def processResponse(self, request, response):
        if response.status == 200 or response.content:
            return
        return CatalogErrorResponse(status=response.status,
                            message=response.message,
                            headers=response.headers,
                            envelopeStatus = self._getEnvelopeStatus(request))

    def processException(self, request, excClass, exception, tb):
        envelopeStatus = self._getEnvelopeStatus(request)
        if isinstance(exception, CatalogError):
            return CatalogErrorResponse(status=exception.status,
                                        message=exception.message,
                                        envelopeStatus = envelopeStatus,
                                        tracebackData = exception.tracebackData,
                                        productCodeData = exception.productCodeData)
        from restlib.http import handler
        response = handler.ExceptionCallback().processException(request,
            excClass, exception, tb)
        return CatalogErrorResponse(status = response.status,
            message = response.message,
            tracebackData = response.content,
            envelopeStatus = envelopeStatus)

    @classmethod
    def _getEnvelopeStatus(cls, request):
        if 'HTTP_X_FLASH_VERSION' not in request.headers:
            return None
        return 200
<commit_msg>Changing field name from message to msg, to shut off python 2.6<commit_after>#
# Copyright (c) 2008 rPath, Inc.
#
from lxml import etree

from catalogService.rest.response import XmlStringResponse
from catalogService import http_codes

class CatalogErrorResponse(XmlStringResponse):
    def __init__(self, status, message, tracebackData='', productCodeData=None, envelopeStatus=None,
                 *args, **kw):
        # See RBL-3818 - flex does not expose the content of a non-200
        # response, so we have to tunnel faults through 200.
        faultNode = etree.Element("fault")
        node = etree.Element("code")
        node.text = str(status)
        faultNode.append(node)

        node = etree.Element("message")
        node.text = message
        faultNode.append(node)

        if tracebackData:
            node = etree.Element("traceback")
            node.text = tracebackData
            faultNode.append(node)

        if productCodeData:
            node = etree.Element("productCode")
            for k, v in productCodeData.iteritems():
                node.set(k, v)
            faultNode.append(node)

        content = etree.tostring(faultNode, pretty_print = True,
            xml_declaration = True, encoding = 'UTF-8')
        # Prefer envelopeStatus if set, otherwise use status
        XmlStringResponse.__init__(self, content=content,
                                   status=envelopeStatus or status,
                                   message=message, *args, **kw)

class CatalogError(Exception):
    """Base class for errors from Cloud Catalog Service"""
    status = http_codes.HTTP_INTERNAL_SERVER_ERROR
    def __init__(self, message=None, status=None, *args, **kw):
        if not message:
            message = self.__class__.__doc__
        if not status:
            status = self.status
        self.status = status
        self.msg = message
        self.tracebackData = kw.get('tracebackData', None)
        self.productCodeData = kw.get('productCodeData', None)

    def __str__(self):
        return self.msg

class InvalidCloudName(CatalogError):
    """Cloud name is not valid"""
    status = http_codes.HTTP_NOT_FOUND

class MissingCredentials(CatalogError):
    """Cloud credentials are not set in rBuilder"""
    status = http_codes.HTTP_BAD_REQUEST

class PermissionDenied(CatalogError):
    """Permission Denied"""
    status = http_codes.HTTP_FORBIDDEN

class ParameterError(CatalogError):
    """Errors were detected in input"""
    status = http_codes.HTTP_BAD_REQUEST
    def __init__(self, message = None):
        CatalogError.__init__(self, message = message)

class ResponseError(CatalogError):
    """Response error from remote cloud service"""
    status = http_codes.HTTP_BAD_REQUEST
    # XXX flex's httpd stack requires we pass a 200 or it will junk the content
    def __init__(self, status, message, body, productCodeData=None):
        # strip any xml tags
        if body.strip().startswith('<?xml'):
            body = ''.join(body.splitlines(True)[1:])
        CatalogError.__init__(self, message, status = status, tracebackData = body, productCodeData = productCodeData)

class CloudExists(CatalogError):
    """Target already exists"""
    status = http_codes.HTTP_CONFLICT

class HttpNotFound(CatalogError):
    """File not found"""
    status = 404

class ErrorMessageCallback(object):
    def processResponse(self, request, response):
        if response.status == 200 or response.content:
            return
        return CatalogErrorResponse(status=response.status,
                            message=response.message,
                            headers=response.headers,
                            envelopeStatus = self._getEnvelopeStatus(request))

    def processException(self, request, excClass, exception, tb):
        envelopeStatus = self._getEnvelopeStatus(request)
        if isinstance(exception, CatalogError):
            return CatalogErrorResponse(status=exception.status,
                                        message=exception.msg,
                                        envelopeStatus = envelopeStatus,
                                        tracebackData = exception.tracebackData,
                                        productCodeData = exception.productCodeData)
        from restlib.http import handler
        response = handler.ExceptionCallback().processException(request,
            excClass, exception, tb)
        return CatalogErrorResponse(status = response.status,
            message = response.message,
            tracebackData = response.content,
            envelopeStatus = envelopeStatus)

    @classmethod
    def _getEnvelopeStatus(cls, request):
        if 'HTTP_X_FLASH_VERSION' not in request.headers:
            return None
        return 200
<|endoftext|>


<commit_before>from django.conf.urls.defaults import patterns, include, url

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    # Examples:
    url(r'^$', 'foodcheck_app.views.home', name='home'),
    url(r'^search/$', views.search),
    # url(r'^search-form/$', views.search_form),
    # url(r'^$', 'foodcheck_app.views.search_form', name='search_form'),
    # url(r'^$', 'foodcheck_app.views.search', name='search'),
    # url(r'^foodcheck/', include('foodcheck.foo.urls')),

    # Uncomment the admin/doc line below to enable admin documentation:
    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),

    # Uncomment the next line to enable the admin:
    url(r'^admin/', include(admin.site.urls)),
)

# vim:expandtab tabstop=8 shiftwidth=4 ts=8 sw=4 softtabstop=4
<commit_msg>Views imported<commit_after>from django.conf.urls.defaults import patterns, include, url

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    # Examples:
    url(r'^$', 'foodcheck_app.views.home', name='home'),
    url(r'^search/$', 'foodcheck_app.views.search', name='search'),
    # url(r'^search-form/$', views.search_form),
    # url(r'^$', 'foodcheck_app.views.search_form', name='search_form'),
    # url(r'^$', 'foodcheck_app.views.search', name='search'),
    # url(r'^foodcheck/', include('foodcheck.foo.urls')),

    # Uncomment the admin/doc line below to enable admin documentation:
    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),

    # Uncomment the next line to enable the admin:
    url(r'^admin/', include(admin.site.urls)),
)

# vim:expandtab tabstop=8 shiftwidth=4 ts=8 sw=4 softtabstop=4
<|endoftext|>


<commit_before># *****************************************************************************
# Copyright (c) 2018 IBM Corporation and other Contributors.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
#
# *****************************************************************************

import getopt
import json
import logging
import os
import signal
import sys
import time

from flask import Flask

try:
    import ibmiotf.application
except ImportError:
    # This part is only required to run the sample from within the samples
    # directory when the module itself is not installed.
    #
    # If you have the module installed, just use "import ibmiotf"
    import os
    import inspect
    cmd_subfolder = os.path.realpath(os.path.abspath(os.path.join(os.path.split(inspect.getfile( inspect.currentframe() ))[0],"../../src")))
    if cmd_subfolder not in sys.path:
        sys.path.insert(0, cmd_subfolder)
    import ibmiotf.application

logger = logging.getLogger('wiotp.deviceconnstate')
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)

TABLE_ROW_TEMPLATE = "%-33s%-30s%s"
NEVER_CONNECTED_REASON  = "Never Connected"

# The dict where the device connection state is stored deviceId -> status payload message
# No locking necessary
deviceConnStateMap = {}

def usage():
    print(
        "deviceStatus: Basic Flask application to report device connection status." + "\n" +
        "\n" +
        "Options: " + "\n" +
        "  -h, --help          Display help information" + "\n" +
        "  -o, --organization  Connect to the specified organization" + "\n" +
        "  -i, --id            Application identifier" + "\n" +
        "  -k, --key           API key used to connect" + "\n" +
        "  -t, --token         Authentication token for the API key specified" + "\n" +
        "  -c, --config        Load application configuration file (ignore -o, -i, -k, -t options)" + "\n"
    )

#####################################################################################
# IoTP Functions
#####################################################################################

def statusCallback(status):
    if status.action == "Disconnect":
        summaryText = "%s %s (%s)" % (status.action, status.clientAddr, status.reason)
    else:
        summaryText = "%s %s" % (status.action, status.clientAddr)

    logger.debug(TABLE_ROW_TEMPLATE % (status.time.isoformat(), status.device, summaryText))
    if status.closeCode == 288:
        # "Action":"Disconnect", "CloseCode":288,"Reason":"The client ID was reused."
        # These status messages are ignored because it is possible this application
        # will receive the disconnect status message after the connect status message,
        # which would led to the wrong status in the map.
        # (Both a connect and disconnect happen when a client ID is reused)
        logger.debug("Ignoring client ID reused")
    else:
        deviceConnStateMap[status.clientId] = status

def interruptHandler(signal, frame):
    client.disconnect()
    sys.exit(0)

#####################################################################################
# Flask Endpoints
#####################################################################################
app = Flask(__name__)

@app.route('/<type>/<id>', methods=['GET'])
def getDeviceStatus(type, id):
    clientId = "d:%s:%s:%s" % (client.organization, type, id)
    if clientId in deviceConnStateMap:
        status = deviceConnStateMap[clientId]
        return json.dumps(status.payload)
    else:
        neverConnectedStatus = {"ClientID": clientId, "Reason": NEVER_CONNECTED_REASON}
        return json.dumps(neverConnectedStatus)

#####################################################################################
# MAIN
#####################################################################################
if __name__ == "__main__":

    try:
        opts, args = getopt.getopt(sys.argv[1:], "h:o:i:k:t:c:p:", ["help", "org=", "id=", "key=", "token=", "config=", "port"])
    except getopt.GetoptError as err:
        print(str(err))
        usage()
        sys.exit(2)

    organization = None
    appId = None
    authMethod = None
    authKey = None
    authToken = None
    configFilePath = None
    port = 5000

    for o, a in opts:
        if o in ("-o", "--organization"):
            organization = a
        elif o in ("-i", "--id"):
            appId = a
        elif o in ("-k", "--key"):
            authMethod = "apikey"
            authKey = a
        elif o in ("-t", "--token"):
            authToken = a
        elif o in ("-c", "--cfg"):
            configFilePath = a
        elif o in ("-p", "--port"):
            port = int(a)
        elif o in ("-h", "--help"):
            usage()
            sys.exit()
        else:
            assert False, "unhandled option" + o

    client = None
    if configFilePath is not None:
        options = ibmiotf.application.ParseConfigFile(configFilePath)
    else:
        options = {"org": organization, "id": appId, "auth-method": authMethod, "auth-key": authKey, "auth-token": authToken}
    try:
        client = ibmiotf.application.Client(options, logHandlers = ch)
        signal.signal(signal.SIGINT, interruptHandler)
        client.connect()
        client.deviceStatusCallback = statusCallback
        client.subscribeToDeviceStatus()
    except ibmiotf.ConfigurationException as e:
        print(str(e))
        sys.exit()
    except ibmiotf.UnsupportedAuthenticationMethod as e:
        print(str(e))
        sys.exit()
    except ibmiotf.ConnectionException as e:
        print(str(e))
        sys.exit()

    app.logger.addHandler(ch)
    app.run(host='0.0.0.0', port=port)
<commit_msg>Use Action instead of Reason<commit_after># *****************************************************************************
# Copyright (c) 2018 IBM Corporation and other Contributors.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
#
# *****************************************************************************

import getopt
import json
import logging
import os
import signal
import sys
import time

from flask import Flask

try:
    import ibmiotf.application
except ImportError:
    # This part is only required to run the sample from within the samples
    # directory when the module itself is not installed.
    #
    # If you have the module installed, just use "import ibmiotf"
    import os
    import inspect
    cmd_subfolder = os.path.realpath(os.path.abspath(os.path.join(os.path.split(inspect.getfile( inspect.currentframe() ))[0],"../../src")))
    if cmd_subfolder not in sys.path:
        sys.path.insert(0, cmd_subfolder)
    import ibmiotf.application

logger = logging.getLogger('wiotp.deviceconnstate')
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)

TABLE_ROW_TEMPLATE = "%-33s%-30s%s"
NEVER_CONNECTED_ACTION  = "Never Connected"

# The dict where the device connection state is stored deviceId -> status payload message
# No locking necessary
deviceConnStateMap = {}

def usage():
    print(
        "deviceStatus: Basic Flask application to report device connection status." + "\n" +
        "\n" +
        "Options: " + "\n" +
        "  -h, --help          Display help information" + "\n" +
        "  -o, --organization  Connect to the specified organization" + "\n" +
        "  -i, --id            Application identifier" + "\n" +
        "  -k, --key           API key used to connect" + "\n" +
        "  -t, --token         Authentication token for the API key specified" + "\n" +
        "  -c, --config        Load application configuration file (ignore -o, -i, -k, -t options)" + "\n"
    )

#####################################################################################
# IoTP Functions
#####################################################################################

def statusCallback(status):
    if status.action == "Disconnect":
        summaryText = "%s %s (%s)" % (status.action, status.clientAddr, status.reason)
    else:
        summaryText = "%s %s" % (status.action, status.clientAddr)

    logger.debug(TABLE_ROW_TEMPLATE % (status.time.isoformat(), status.device, summaryText))
    if status.closeCode == 288:
        # "Action":"Disconnect", "CloseCode":288,"Reason":"The client ID was reused."
        # These status messages are ignored because it is possible this application
        # will receive the disconnect status message after the connect status message,
        # which would led to the wrong status in the map.
        # (Both a connect and disconnect happen when a client ID is reused)
        logger.debug("Ignoring client ID reused")
    else:
        deviceConnStateMap[status.clientId] = status

def interruptHandler(signal, frame):
    client.disconnect()
    sys.exit(0)

#####################################################################################
# Flask Endpoints
#####################################################################################
app = Flask(__name__)

@app.route('/<type>/<id>', methods=['GET'])
def getDeviceStatus(type, id):
    clientId = "d:%s:%s:%s" % (client.organization, type, id)
    if clientId in deviceConnStateMap:
        status = deviceConnStateMap[clientId]
        return json.dumps(status.payload)
    else:
        neverConnectedStatus = {"ClientID": clientId, "Action": NEVER_CONNECTED_ACTION}
        return json.dumps(neverConnectedStatus)

#####################################################################################
# MAIN
#####################################################################################
if __name__ == "__main__":

    try:
        opts, args = getopt.getopt(sys.argv[1:], "h:o:i:k:t:c:p:", ["help", "org=", "id=", "key=", "token=", "config=", "port"])
    except getopt.GetoptError as err:
        print(str(err))
        usage()
        sys.exit(2)

    organization = None
    appId = None
    authMethod = None
    authKey = None
    authToken = None
    configFilePath = None
    port = 5000

    for o, a in opts:
        if o in ("-o", "--organization"):
            organization = a
        elif o in ("-i", "--id"):
            appId = a
        elif o in ("-k", "--key"):
            authMethod = "apikey"
            authKey = a
        elif o in ("-t", "--token"):
            authToken = a
        elif o in ("-c", "--cfg"):
            configFilePath = a
        elif o in ("-p", "--port"):
            port = int(a)
        elif o in ("-h", "--help"):
            usage()
            sys.exit()
        else:
            assert False, "unhandled option" + o

    client = None
    if configFilePath is not None:
        options = ibmiotf.application.ParseConfigFile(configFilePath)
    else:
        options = {"org": organization, "id": appId, "auth-method": authMethod, "auth-key": authKey, "auth-token": authToken}
    try:
        client = ibmiotf.application.Client(options, logHandlers = ch)
        signal.signal(signal.SIGINT, interruptHandler)
        client.connect()
        client.deviceStatusCallback = statusCallback
        client.subscribeToDeviceStatus()
    except ibmiotf.ConfigurationException as e:
        print(str(e))
        sys.exit()
    except ibmiotf.UnsupportedAuthenticationMethod as e:
        print(str(e))
        sys.exit()
    except ibmiotf.ConnectionException as e:
        print(str(e))
        sys.exit()

    app.logger.addHandler(ch)
    app.run(host='0.0.0.0', port=port)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from pymisp import PyMISP
from keys import misp_url, misp_key, misp_verifycert
import argparse
import tools
import timing

def formattingDataframe(dataframe, dates, NanValue):
    dataframe.reverse()
    dates.reverse()
    dataframe = tools.concat(dataframe)
    dataframe = tools.renameColumns(dataframe, dates)
    dataframe = tools.replaceNaN(dataframe, 0)
    return dataframe

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Show the evolution of trend of tags.')
    parser.add_argument("-p", "--period", help='Define the studied period. Can be the past year (y), month (m) or week (w). Week is the default value if no valid value is given.')
    parser.add_argument("-a", "--accuracy", help='Define the accuracy of the splits on the studied period. Can be per month (m) -for year only-, week (w) -month only- or day (d). The default value is always the biggest available.')
    parser.add_argument("-o", "--order", type=int, help='Define the accuracy of the curve fitting. Default value is 3')

    args = parser.parse_args()

    misp = PyMISP(misp_url, misp_key, misp_verifycert)

    if args.period == "y":
        if args.accuracy == "d":
            split = 360
            size = 1
        else:
            split = 12
            size = 30
        last = '360d'
        title = 'Tags repartition over the last 360 days'
    elif args.period == "m":
        if args.accuracy == "d":
            split = 28
            size = 1
        else:
            split = 4
            size = 7
        last = '28d'
        title = 'Tags repartition over the last 28 days'
    else:
        split = 7
        size = 1
        last = '7d'
        title = 'Tags repartition over the last 7 days'

    result = misp.search(last=last, metadata=True)
    if 'response' in result:
        events = tools.eventsListBuildFromArray(result)
        result = []
        dates = []
        enddate = tools.getToday()
        colourDict = {}
        faketag = False

        for i in range(split):
            begindate = tools.getNDaysBefore(enddate, size)
            dates.append(str(enddate.date()))
            eventstemp = tools.selectInRange(events, begin=begindate, end=enddate)
            if eventstemp is not None:
                tags = tools.tagsListBuild(eventstemp)
                if tags is not None:
                    tools.createDictTagsColour(colourDict, tags)
                    result.append(tools.getNbOccurenceTags(tags))
                else:
                    result.append(tools.createFakeEmptyTagsSeries())
                    faketag = True
            else:
                result.append(tools.createFakeEmptyTagsSeries())
                faketag = True
            enddate = begindate

        result = formattingDataframe(result, dates, 0)
        if faketag:
            result = tools.removeFaketagRow(result)

        taxonomies, emptyOther = tools.getTaxonomies(tools.getCopyDataframe(result))

        tools.tagsToLineChart(tools.getCopyDataframe(result), title, dates, colourDict)
        tools.tagstrendToLineChart(tools.getCopyDataframe(result), title, dates, split, colourDict)
        tools.tagsToTaxoLineChart(tools.getCopyDataframe(result), title, dates, colourDict, taxonomies, emptyOther)
        tools.tagstrendToTaxoLineChart(tools.getCopyDataframe(result), title, dates, split, colourDict, taxonomies, emptyOther)
        if args.order is None:
            args.order = 3
        tools.tagsToPolyChart(tools.getCopyDataframe(result), split, colourDict, taxonomies, emptyOther, args.order)
        tools.createVisualisation(taxonomies)

    else:
        print('There is no event during the studied period')
<commit_msg>remove test import<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from pymisp import PyMISP
from keys import misp_url, misp_key, misp_verifycert
import argparse
import tools

def formattingDataframe(dataframe, dates, NanValue):
    dataframe.reverse()
    dates.reverse()
    dataframe = tools.concat(dataframe)
    dataframe = tools.renameColumns(dataframe, dates)
    dataframe = tools.replaceNaN(dataframe, 0)
    return dataframe

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Show the evolution of trend of tags.')
    parser.add_argument("-p", "--period", help='Define the studied period. Can be the past year (y), month (m) or week (w). Week is the default value if no valid value is given.')
    parser.add_argument("-a", "--accuracy", help='Define the accuracy of the splits on the studied period. Can be per month (m) -for year only-, week (w) -month only- or day (d). The default value is always the biggest available.')
    parser.add_argument("-o", "--order", type=int, help='Define the accuracy of the curve fitting. Default value is 3')

    args = parser.parse_args()

    misp = PyMISP(misp_url, misp_key, misp_verifycert)

    if args.period == "y":
        if args.accuracy == "d":
            split = 360
            size = 1
        else:
            split = 12
            size = 30
        last = '360d'
        title = 'Tags repartition over the last 360 days'
    elif args.period == "m":
        if args.accuracy == "d":
            split = 28
            size = 1
        else:
            split = 4
            size = 7
        last = '28d'
        title = 'Tags repartition over the last 28 days'
    else:
        split = 7
        size = 1
        last = '7d'
        title = 'Tags repartition over the last 7 days'

    result = misp.search(last=last, metadata=True)
    if 'response' in result:
        events = tools.eventsListBuildFromArray(result)
        result = []
        dates = []
        enddate = tools.getToday()
        colourDict = {}
        faketag = False

        for i in range(split):
            begindate = tools.getNDaysBefore(enddate, size)
            dates.append(str(enddate.date()))
            eventstemp = tools.selectInRange(events, begin=begindate, end=enddate)
            if eventstemp is not None:
                tags = tools.tagsListBuild(eventstemp)
                if tags is not None:
                    tools.createDictTagsColour(colourDict, tags)
                    result.append(tools.getNbOccurenceTags(tags))
                else:
                    result.append(tools.createFakeEmptyTagsSeries())
                    faketag = True
            else:
                result.append(tools.createFakeEmptyTagsSeries())
                faketag = True
            enddate = begindate

        result = formattingDataframe(result, dates, 0)
        if faketag:
            result = tools.removeFaketagRow(result)

        taxonomies, emptyOther = tools.getTaxonomies(tools.getCopyDataframe(result))

        tools.tagsToLineChart(tools.getCopyDataframe(result), title, dates, colourDict)
        tools.tagstrendToLineChart(tools.getCopyDataframe(result), title, dates, split, colourDict)
        tools.tagsToTaxoLineChart(tools.getCopyDataframe(result), title, dates, colourDict, taxonomies, emptyOther)
        tools.tagstrendToTaxoLineChart(tools.getCopyDataframe(result), title, dates, split, colourDict, taxonomies, emptyOther)
        if args.order is None:
            args.order = 3
        tools.tagsToPolyChart(tools.getCopyDataframe(result), split, colourDict, taxonomies, emptyOther, args.order)
        tools.createVisualisation(taxonomies)

    else:
        print('There is no event during the studied period')
<|endoftext|>


<commit_before>__author__ = 'stephenlenzi'

import sys
import numpy as np
from scipy import ndimage
from skimage import segmentation

class MaskGenerator(object):
    """
    Parent class for generating masklists and manually correcting them
    """
    def __init__(self, blob_image, raw_image, com=None):

        self.raw_image = raw_image
        self.blob_image = blob_image

        if com is None:
            self.com = get_blob_centers(self.blob_image)
        else:
            self.com = com

        self.putative_nuclei_image, _ = remove_small_blobs(self.blob_image, self.com)  # remove non-overlapping centers

    def append_com(self, c):
        x = self.com.tolist()
        x.append(c)
        self.com = np.array(x)

    def remove_com(self, c):
        x = self.com.tolist()
        x.remove(c)
        self.com = np.array(x)

    def update(self):
        pass

    def create_roi_masks(self, seg_type):
        pass

class DonutCells(MaskGenerator):
    """
    Child class to handle GCaMP data - i.e. donut shaped rois, with a dark nucleus
    """
    def __init__(self, raw_image, putative_nuclei_image, putative_somata_image, com=None):
        super(DonutCells, self).__init__(putative_nuclei_image, raw_image, com)
        self.putative_somata_image = putative_somata_image
        self.putative_nuclei_image = ndimage.binary_fill_holes(self.putative_nuclei_image)
        self.watershed_image = np.logical_or(self.putative_nuclei_image, self.putative_somata_image)
        self.segmentation_labels = calculate_distance(self.watershed_image, self.com)
        self.labelled_nuclei = calculate_distance(self.putative_nuclei_image, self.com)
        self.roi_masks = create_roi_masks(self.com, self.putative_nuclei_image, self.putative_somata_image)

    def update(self):
        self.putative_nuclei_image, _ = remove_small_blobs(self.putative_nuclei_image, self.com)
        self.segmentation_labels = calculate_distance(self.watershed_image, self.com)
        self.roi_masks = create_roi_masks(self.com, self.putative_nuclei_image, self.putative_somata_image)

class BlobCells(MaskGenerator):
    """
    Child class for blob cells - i.e. bolus loaded cells. Nucleus also fluorescent
    """

    def __init__(self, segmentation_layer, raw_image):
        super(BlobCells, self).__init__(segmentation_layer, raw_image)
        self.watershed_image = self.putative_nuclei_image
        self.segmentation_labels = calculate_distance(self.watershed_image, self.com)
        self.roi_mask_list = self.create_roi_masks()

    def create_roi_masks(self):
        roi_mask_list = []
        for i in range(np.max(self.segmentation_labels)):
            idx = np.where(self.segmentation_labels*self.putative_nuclei_image == i+1)
            x = np.vstack((idx[0], idx[1]))
            m = np.reshape(x.T, (len(idx[0]), 2))
            roi_mask_list.append(m)

        return roi_mask_list

def get_blob_centers(segmentation_layer, iterations=3):
    """
    Determine the centers of each object in an image

    ::param segmentation_layer: NxM ndarray image mask of all target objects
    ::param iterations: threshold for removal of small non-target objects
    ::returns com: a np ndarray of x,y coordinates for the center of each target object

    """

    segmentation_layer = ndimage.binary_opening(segmentation_layer, iterations=iterations)  # remove small objects
    labels, label_number = ndimage.measurements.label(segmentation_layer)  # label remaining blobs

    com = np.zeros((label_number, 2))

    for i in range(label_number):
        idx = np.where(labels == i+1)  # calculate the center of mass for each blob
        com[i, 1] = np.mean(idx[1].astype(float))  # must be float
        com[i, 0] = np.mean(idx[0].astype(float))  # must be float

    return com

def remove_small_blobs(segmentation_layer, com):
    """
    removes non-overlapping pixel-islands and cell centres (coms)
    ::param segmentation_layer: NxM ndarray image mask of all target objects
    ::param com: a np ndarray of x,y coordinates for the center of each target object
    ::return updated_labels:
    """
    labels, label_number = ndimage.label(segmentation_layer)  # label all pixel islands
    updated_labels = np.zeros_like(labels)
    coms_to_keep = np.zeros(len(com))  # holder

    for i in range(label_number):
        idx = np.where(labels == i+1)
        for j, c in enumerate(com.astype(int)):
            if labels[c[0], c[1]] == i+1:  # if the com is in the blob
                updated_labels[idx] = 1  # add the blob
                coms_to_keep[j] = 1  # add the com

    com_index = np.where(coms_to_keep == 1)
    updated_com = com[com_index]

    return updated_labels, updated_com

def calculate_distance(image, com):
    """
    ::param image: a binarised image to be segmented
    ::param com: the centres that will define the maxima of the watershed segmentation
    """
    # random walk segmentation of 2D image-mask array
    distance = ndimage.distance_transform_edt(np.abs(image-1))
    local_maxi = np.zeros_like(image)

    for c in com:
        local_maxi[int(c[0]), int(c[1])] = 1

    markers = ndimage.label(local_maxi)[0]
    segmentation_labels = segmentation.random_walker(distance, markers, beta=60)

    return segmentation_labels

def blob_labels(com, blob_image):
    """
    label nuclei with segmentation - so labels are in the same order as the outer layer
    """

    image = np.abs(blob_image-1)
    distance = ndimage.distance_transform_edt(np.abs(image-1))
    local_maxi = np.zeros_like(image)
    for c in com:
        local_maxi[int(c[0]), int(c[1])] = 1
    markers = ndimage.label(local_maxi)[0]
    segmented_blobs = segmentation.random_walker(distance, markers, beta=20)

    return segmented_blobs, distance

def create_roi_masks(com, putative_nuclei_image, putative_somata_image=None, radius=3):
        """
        create roi masks for the outer segment of the cell (i.e. soma)
        ::param radius: limits the size of the mask
        """
        roi_mask_list = []

        if putative_somata_image is None:
            putative_somata_image = np.zeros_like(putative_nuclei_image)
        putative_nuclei_image = remove_small_blobs(putative_nuclei_image,com)[0]
        watershed_image = np.logical_or(putative_nuclei_image, putative_somata_image)
        labelled_watershed = calculate_distance(watershed_image, com)
        labelled_putative_somata = putative_somata_image*labelled_watershed
        labelled_putative_nuclei = calculate_distance(putative_nuclei_image,
                                                      com)*putative_nuclei_image  # nuclei need their own watershed

        for i in range(np.max(labelled_putative_somata)):  # for each nucleus

            # calculate the distance away from the nucleus boundary

            distance_from_blob_centre = ndimage.distance_transform_edt(labelled_putative_nuclei != i+1)
            bool_mask = np.ones_like(labelled_putative_nuclei)
            bool_mask[distance_from_blob_centre > radius] = 0
            bool_mask[distance_from_blob_centre == 0] = 0

            # take all indices within the radius number of pixels of the nucleus boundary

            idx = np.where(labelled_putative_somata*bool_mask == i+1)
            x = np.vstack((idx[0], idx[1]))
            m = np.reshape(x.T, (len(idx[0]), 2))
            roi_mask_list.append(m)

        return roi_mask_list
<commit_msg>renamed com to center_of_mass<commit_after>__author__ = 'stephenlenzi'

import sys
import numpy as np
from scipy import ndimage
from skimage import segmentation

class MaskGenerator(object):
    """
    Parent class for generating masklists and manually correcting them
    """
    def __init__(self, blob_image, raw_image, center_of_mass=None):

        self.raw_image = raw_image
        self.blob_image = blob_image

        if center_of_mass is None:
            self.center_of_mass = get_blob_centers(self.blob_image)
        else:
            self.center_of_mass = center_of_mass

        self.putative_nuclei_image, _ = remove_small_blobs(self.blob_image, self.center_of_mass)  # remove non-overlapping centers

    def append_com(self, c):
        x = self.center_of_mass.tolist()
        x.append(c)
        self.center_of_mass = np.array(x)

    def remove_com(self, c):
        x = self.center_of_mass.tolist()
        x.remove(c)
        self.center_of_mass = np.array(x)

    def update(self):
        pass

    def create_roi_masks(self, seg_type):
        pass

class DonutCells(MaskGenerator):
    """
    Child class to handle GCaMP data - i.e. donut shaped rois, with a dark nucleus
    """
    def __init__(self, raw_image, putative_nuclei_image, putative_somata_image, center_of_mass=None):
        super(DonutCells, self).__init__(putative_nuclei_image, raw_image, center_of_mass)
        self.putative_somata_image = putative_somata_image
        self.putative_nuclei_image = ndimage.binary_fill_holes(self.putative_nuclei_image)
        self.watershed_image = np.logical_or(self.putative_nuclei_image, self.putative_somata_image)
        self.segmentation_labels = calculate_distance(self.watershed_image, self.center_of_mass)
        self.labelled_nuclei = calculate_distance(self.putative_nuclei_image, self.center_of_mass)
        self.roi_masks = create_roi_masks(self.center_of_mass, self.putative_nuclei_image, self.putative_somata_image)

    def update(self):
        self.putative_nuclei_image, _ = remove_small_blobs(self.putative_nuclei_image, self.center_of_mass)
        self.segmentation_labels = calculate_distance(self.watershed_image, self.center_of_mass)
        self.roi_masks = create_roi_masks(self.center_of_mass, self.putative_nuclei_image, self.putative_somata_image)

class BlobCells(MaskGenerator):
    """
    Child class for blob cells - i.e. bolus loaded cells. Nucleus also fluorescent
    """

    def __init__(self, segmentation_layer, raw_image):
        super(BlobCells, self).__init__(segmentation_layer, raw_image)
        self.watershed_image = self.putative_nuclei_image
        self.segmentation_labels = calculate_distance(self.watershed_image, self.center_of_mass)
        self.roi_mask_list = self.create_roi_masks()

    def create_roi_masks(self):
        roi_mask_list = []
        for i in range(np.max(self.segmentation_labels)):
            idx = np.where(self.segmentation_labels*self.putative_nuclei_image == i+1)
            x = np.vstack((idx[0], idx[1]))
            m = np.reshape(x.T, (len(idx[0]), 2))
            roi_mask_list.append(m)

        return roi_mask_list

def get_blob_centers(segmentation_layer, iterations=3):
    """
    Determine the centers of each object in an image

    ::param segmentation_layer: NxM ndarray image mask of all target objects
    ::param iterations: threshold for removal of small non-target objects
    ::returns com: a np ndarray of x,y coordinates for the center of each target object

    """

    segmentation_layer = ndimage.binary_opening(segmentation_layer, iterations=iterations)  # remove small objects
    labels, label_number = ndimage.measurements.label(segmentation_layer)  # label remaining blobs

    com = np.zeros((label_number, 2))

    for i in range(label_number):
        idx = np.where(labels == i+1)  # calculate the center of mass for each blob
        com[i, 1] = np.mean(idx[1].astype(float))  # must be float
        com[i, 0] = np.mean(idx[0].astype(float))  # must be float

    return com

def remove_small_blobs(segmentation_layer, com):
    """
    removes non-overlapping pixel-islands and cell centres (coms)
    ::param segmentation_layer: NxM ndarray image mask of all target objects
    ::param com: a np ndarray of x,y coordinates for the center of each target object
    ::return updated_labels:
    """
    labels, label_number = ndimage.label(segmentation_layer)  # label all pixel islands
    updated_labels = np.zeros_like(labels)
    coms_to_keep = np.zeros(len(com))  # holder

    for i in range(label_number):
        idx = np.where(labels == i+1)
        for j, c in enumerate(com.astype(int)):
            if labels[c[0], c[1]] == i+1:  # if the com is in the blob
                updated_labels[idx] = 1  # add the blob
                coms_to_keep[j] = 1  # add the com

    com_index = np.where(coms_to_keep == 1)
    updated_com = com[com_index]

    return updated_labels, updated_com

def calculate_distance(image, com):
    """
    ::param image: a binarised image to be segmented
    ::param com: the centres that will define the maxima of the watershed segmentation
    """
    # random walk segmentation of 2D image-mask array
    distance = ndimage.distance_transform_edt(np.abs(image-1))
    local_maxi = np.zeros_like(image)

    for c in com:
        local_maxi[int(c[0]), int(c[1])] = 1

    markers = ndimage.label(local_maxi)[0]
    segmentation_labels = segmentation.random_walker(distance, markers, beta=60)

    return segmentation_labels

def blob_labels(com, blob_image):
    """
    label nuclei with segmentation - so labels are in the same order as the outer layer
    """

    image = np.abs(blob_image-1)
    distance = ndimage.distance_transform_edt(np.abs(image-1))
    local_maxi = np.zeros_like(image)
    for c in com:
        local_maxi[int(c[0]), int(c[1])] = 1
    markers = ndimage.label(local_maxi)[0]
    segmented_blobs = segmentation.random_walker(distance, markers, beta=20)

    return segmented_blobs, distance

def create_roi_masks(com, putative_nuclei_image, putative_somata_image=None, radius=3):
        """
        create roi masks for the outer segment of the cell (i.e. soma)
        ::param radius: limits the size of the mask
        """
        roi_mask_list = []

        if putative_somata_image is None:
            putative_somata_image = np.zeros_like(putative_nuclei_image)
        putative_nuclei_image = remove_small_blobs(putative_nuclei_image,com)[0]
        watershed_image = np.logical_or(putative_nuclei_image, putative_somata_image)
        labelled_watershed = calculate_distance(watershed_image, com)
        labelled_putative_somata = putative_somata_image*labelled_watershed
        labelled_putative_nuclei = calculate_distance(putative_nuclei_image,
                                                      com)*putative_nuclei_image  # nuclei need their own watershed

        for i in range(np.max(labelled_putative_somata)):  # for each nucleus

            # calculate the distance away from the nucleus boundary

            distance_from_blob_centre = ndimage.distance_transform_edt(labelled_putative_nuclei != i+1)
            bool_mask = np.ones_like(labelled_putative_nuclei)
            bool_mask[distance_from_blob_centre > radius] = 0
            bool_mask[distance_from_blob_centre == 0] = 0

            # take all indices within the radius number of pixels of the nucleus boundary

            idx = np.where(labelled_putative_somata*bool_mask == i+1)
            x = np.vstack((idx[0], idx[1]))
            m = np.reshape(x.T, (len(idx[0]), 2))
            roi_mask_list.append(m)

        return roi_mask_list
<|endoftext|>


<commit_before><commit_msg>isEventListenerEnabled() should return True by default<commit_after><|endoftext|>