<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# ------------------------------------------------------------------------------
#
# Author: Gabriele Girelli
# Email: gigi.ga90@gmail.com
# Version: 1.0.0
# Description: script to interface with the pygpseq package.
# ------------------------------------------------------------------------------

# DEPENDENCIES =================================================================

import argparse
import numpy as np
import os
import pandas as pd
import pygpseq as gp
import sys

# PARAMETERS ===================================================================

# Add script description
parser = argparse.ArgumentParser(
	description = """Run pygpseq-based analysis."""
)

# Positional parameters
parser.add_argument('inDir', type = str, nargs = 1,
	help = """Path to input directory, containing single-condition directories
	with TIF files.""")
parser.add_argument('outDir', type = str, nargs = 1,
	help = """Path to output directory, must be different from the input
	directory.""")

# Optional parameters
parser.add_argument('--skip', type = str, nargs = '*',
	help = """Space-separated phases to be skipped.
	Use -- after the last one.""",
	choices = ['inst', 'seg', 'an', 'box', 'plot', 'report'])
parser.add_argument('-l', '--logpath', type = str, nargs = 1,
	help = """Path to log file. By default: outDir/log""", metavar = 'log')
parser.add_argument('-a', '--aspect', type = float, nargs = 3,
	help = """Physical size of Z, Y and X voxel sides.
	Default: 300.0 216.6 216.6""",
	metavar = ('Z', 'Y', 'X'), default = [300., 216.6, 216.6])
parser.add_argument('-d', '--dna-channels', type = str, nargs = '+',
	help = """Space-separated names of DNA staining channels.
	Use -- after the last one.""",
	default = ['dapi'], metavar = 'dna_name')
parser.add_argument('-s', '--sig-channels', type = str, nargs = '+',
	help = """Space-separated names of GPSeq signal channels.
	Use -- after the last one.""",
	default = ['cy5', 'tmr'], metavar = 'sig_name')
parser.add_argument('-z', '--min-z', type = float, nargs = 1,
	help = """If lower than 1, minimum fraction of stack, if higher than 1,
	minimum number of slices to be occupied by a nucleus""",
	default = [.25], metavar = 'min_z')
parser.add_argument('--seg-type', type = str, nargs = 1,
	help = """Segmentation type. Default: 3d""",
	choices = ['sum_proj', 'max_proj', '3d'], default = ['3d'])
parser.add_argument('--an-type', type = str, nargs = 1,
	help = """Analysis type. Default: mid""",
	choices = ['sum_proj', 'max_proj', '3d', 'mid'],
	default = ['mid'])
parser.add_argument('--mid-type', type = str, nargs = 1,
	help = """Method for mid-section selection.""",
	choices = ['central', 'largest', 'maxIsum'],
	default = ['largest'])
parser.add_argument('--nuclear-sel', type = str, nargs = '*',
	help = """Space-separated features for nuclear selection.
	Use -- after the last one. Default: flat_size sumI""",
	choices = ['size', 'surf', 'shape', 'sumI', 'meanI', 'flat_size'],
	default = ['flat_size', 'sumI'])
parser.add_argument('--description', type = str, nargs = '*',
	help = """Space separated condition:description couples.
	'condition' are the name of condition folders.
	'description' are descriptive labels used in plots instead of folder names.
	Use -- after the last one.""")
parser.add_argument('-t', '--threads', metavar = 'ncores', type = int,
	nargs = 1, default = [1],
	help = """Number of threads to be used for parallelization. Increasing the
	number of threads might increase the required amount of RAM.""")
parser.add_argument('--note', type = str, nargs = 1,
	help = """Dataset/Analysis description. Use double quotes.""")
regexp = "^(?P<channel_name>[^/]*)\.(?P<channel_id>channel[0-9]+)"
regexp += "\.(?P<series_id>series[0-9]+)(?P<ext>(_cmle)?\.tif)$"
parser.add_argument('--regexp', type = str, nargs = 1,
	help = """Advanced. Regular expression to identify tif images.""",
	default = [regexp])

# Flag parameters
parser.add_argument('-r', '--rescale-deconvolved', action = 'store_const',
	help = """Perform rescaling of deconvolved images. Requires Huygens
	Professional v4.5 log file for an image to be rescaled.""",
	const = True, default = False)
parser.add_argument('-n', '--normalize-distance', action = 'store_const',
	help = """Perform distance normalization. Necessary to compare nuclei
	with different radius.""",
	const = True, default = False)
parser.add_argument('-u', '--DEBUG-MODE', action = 'store_const',
	help = """Debugging mode.""",
	const = True, default = False)

# Parse arguments
args = parser.parse_args()

# FUNCTION =====================================================================

def ask(q):
	"""Asks for confirmation. Aborts otherwise.

	Args:
		q (string): question.
	"""

	answer = ''
	while not answer.lower() in ['y', 'n']:
		print("%s %s" % (q, "(y/n)"))
		answer = raw_input()

		if 'n' == answer.lower():
			sys.exit("Aborted.\n")
		elif not 'y' == answer.lower():
			print("Please, answer 'y' or 'n'.\n")

# RUN ==========================================================================

# Create pyGPSeq analyzer instance
gpi = gp.Main(ncores = args.threads[0])

# Steps to be skipped
dskip = {
	'inst' : 1,
	'seg' : 2,
	'an' : 3,
	'box' : 3.5,
	'plot' : 4,
	'report' : 5
}
if not None is args.skip:
	gpi.skip = [dskip[e] for e in args.skip]

# Channel names
gpi.sig_names = tuple(args.sig_channels)
gpi.dna_names = tuple(args.dna_channels)

# Data directory
gpi.basedir = args.inDir[0]

# Output directory
gpi.outdir = args.outDir[0]

# Segmentation type for nuclear identification
dseg = {
	'sum_proj' : 0,
	'max_proj' : 1,
	'3d' : 2
}
gpi.seg_type = dseg[args.seg_type[0]]

# Single-nucleus analysis type
dan = {
	'sum_proj' : 0,
	'max_proj' : 1,
	'3d' : 2,
	'mid' : 3
}
gpi.an_type = dan[args.an_type[0]]

# Middle-section selection method
dmid = {
'central' : 0,
'largest' : 1,
'maxIsum' : 2
}
gpi.mid_type = dmid[args.mid_type[0]]

# Voxel aspect proportions (or sizes, ZYX)
gpi.aspect = tuple(args.aspect)

# Minimum percentage of stack to be occupied by a cell
gpi.min_z_size = args.min_z[0]

# Nuclear selection
dnsel = {
	'size' : 0,
	'surf' : 1,
	'shape' : 2,
	'sumI' : 3,
	'meanI' : 4,
	'flat_size' : 5
}
gpi.nsf = tuple([dnsel[e] for e in args.nuclear_sel])

# Regular expression to identify image files
gpi.reg = args.regexp[0]

# Where to save the run log
if None is args.logpath:
	gpi.logpath = args.outDir[0] + '/' + gpi.gen_log_name()
else:
	gpi.logpath = args.logpath[0]

# Perform deconvolved image rescaling?
gpi.rescale_deconvolved = args.rescale_deconvolved

# Normalize distance?
gpi.normalize_distance = args.normalize_distance

# Better condition naming
if not None is args.description:
	for descr in args.description:
		c, d = descr.split(':')
		gpi.cdescr[c] = d
readable_cdescr = [str(k) + ' => ' + str(v) for (k,v) in gpi.cdescr.items()]
if 0 == len(readable_cdescr):
	readable_cdescr = ["*NONE*"]

# Notes
if not None is args.note:
	gpi.notes = args.note[0]

# Debugging mode
gpi.debugging = args.DEBUG_MODE

# Show current settings
os.system('clear')
print("""
---------- SETTING:  VALUE ----------

   Input directory:  """+gpi.basedir+"""
  Output directory:  """+gpi.outdir+"""
          Log file:  """+gpi.logpath+"""

     Skipped steps:  """+str(args.skip)+"""

      DNA channels:  """+str(gpi.dna_names)+"""
   Signal channels:  """+str(gpi.sig_names)+"""

      Segmentation:  """+args.seg_type[0]+"""
          Analysis:  """+args.an_type[0]+"""
    Middle section:  """+args.mid_type[0]+"""

Voxel aspect (ZYX):  """+str(gpi.aspect)+"""
 Minimum Z portion:  """+str(gpi.min_z_size)+"""

  Condition descr.:  """+"\n                     ".join(readable_cdescr)+"""

           Threads:  """+str(gpi.ncores)+"""
              Note:  """+gpi.notes+"""

            Regexp: """+gpi.reg+"""

   Rescale deconv.:  """+str(gpi.rescale_deconvolved)+"""
   Normalize dist.:  """+str(gpi.normalize_distance)+"""
         Debug mod:  """+str(gpi.debugging)+"""

""")

# Ask for confirmation
ask('Confirm settings and proceed.')

# Start the analysis
gpi = gpi.run()

# End --------------------------------------------------------------------------

################################################################################
<commit_msg>Updated pygpseq-run.py help page.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# ------------------------------------------------------------------------------
#
# Author: Gabriele Girelli
# Email: gigi.ga90@gmail.com
# Version: 1.0.0
# Description: script to interface with the pygpseq package.
# ------------------------------------------------------------------------------

# DEPENDENCIES =================================================================

import argparse
import numpy as np
import os
import pandas as pd
import pygpseq as gp
import sys

# PARAMETERS ===================================================================

# Add script description
parser = argparse.ArgumentParser(
	description = """Run pygpseq-based analysis."""
)

# Positional parameters
parser.add_argument('inDir', type = str, nargs = 1,
	help = """Path to input directory, containing single-condition directories
	with TIF files.""")
parser.add_argument('outDir', type = str, nargs = 1,
	help = """Path to output directory, must be different from the input
	directory.""")

# Optional parameters
parser.add_argument('--skip', type = str, nargs = '*',
	help = """Space-separated phases to be skipped.
	Use -- after the last one.""",
	choices = ['inst', 'seg', 'an', 'box', 'plot', 'report'])
parser.add_argument('-l', '--logpath', type = str, nargs = 1,
	help = """Path to log file. By default: outDir/log""", metavar = 'log')
parser.add_argument('-a', '--aspect', type = float, nargs = 3,
	help = """Physical size of Z, Y and X voxel sides.
	Default: 300.0 216.6 216.6""",
	metavar = ('Z', 'Y', 'X'), default = [300., 216.6, 216.6])
parser.add_argument('-d', '--dna-channels', type = str, nargs = '+',
	help = """Space-separated names of DNA staining channels.
	Use -- after the last one.""",
	default = ['dapi'], metavar = 'dna_name')
parser.add_argument('-s', '--sig-channels', type = str, nargs = '+',
	help = """Space-separated names of GPSeq signal channels.
	Use -- after the last one.""",
	default = ['cy5', 'tmr'], metavar = 'sig_name')
parser.add_argument('-z', '--min-z', type = float, nargs = 1,
	help = """If lower than 1, minimum fraction of stack, if higher than 1,
	minimum number of slices to be occupied by a nucleus. Default: .25""",
	default = [.25], metavar = 'min_z')
parser.add_argument('--seg-type', type = str, nargs = 1,
	help = """Segmentation type. Default: 3d""",
	choices = ['sum_proj', 'max_proj', '3d'], default = ['3d'])
parser.add_argument('--an-type', type = str, nargs = 1,
	help = """Analysis type. Default: mid""",
	choices = ['sum_proj', 'max_proj', '3d', 'mid'],
	default = ['mid'])
parser.add_argument('--mid-type', type = str, nargs = 1,
	help = """Method for mid-section selection. Default: largest""",
	choices = ['central', 'largest', 'maxIsum'],
	default = ['largest'])
parser.add_argument('--nuclear-sel', type = str, nargs = '*',
	help = """Space-separated features for nuclear selection.
	Use -- after the last one. Default: flat_size sumI""",
	choices = ['size', 'surf', 'shape', 'sumI', 'meanI', 'flat_size'],
	default = ['flat_size', 'sumI'])
parser.add_argument('--description', type = str, nargs = '*',
	help = """Space separated condition:description couples.
	'condition' are the name of condition folders.
	'description' are descriptive labels used in plots instead of folder names.
	Use -- after the last one.""")
parser.add_argument('-t', '--threads', metavar = 'nthreads', type = int,
	nargs = 1, default = [1],
	help = """Number of threads to be used for parallelization. Increasing the
	number of threads might increase the required amount of RAM.""")
parser.add_argument('--note', type = str, nargs = 1,
	help = """Dataset/Analysis description. Use double quotes.""")
regexp = "^(?P<channel_name>[^/]*)\.(?P<channel_id>channel[0-9]+)"
regexp += "\.(?P<series_id>series[0-9]+)(?P<ext>(_cmle)?\.tif)$"
parser.add_argument('--regexp', type = str, nargs = 1,
	help = """Advanced. Regular expression to identify tif images.""",
	default = [regexp])

# Flag parameters
parser.add_argument('-r', '--rescale-deconvolved', action = 'store_const',
	help = """Perform rescaling of deconvolved images. Requires Huygens
	Professional v4.5 log file for an image to be rescaled.""",
	const = True, default = False)
parser.add_argument('-n', '--normalize-distance', action = 'store_const',
	help = """Perform distance normalization. Necessary to compare nuclei
	with different radius.""",
	const = True, default = False)
parser.add_argument('-u', '--DEBUG-MODE', action = 'store_const',
	help = """Debugging mode.""",
	const = True, default = False)

# Parse arguments
args = parser.parse_args()

# FUNCTION =====================================================================

def ask(q):
	"""Asks for confirmation. Aborts otherwise.

	Args:
		q (string): question.
	"""

	answer = ''
	while not answer.lower() in ['y', 'n']:
		print("%s %s" % (q, "(y/n)"))
		answer = raw_input()

		if 'n' == answer.lower():
			sys.exit("Aborted.\n")
		elif not 'y' == answer.lower():
			print("Please, answer 'y' or 'n'.\n")

# RUN ==========================================================================

# Create pyGPSeq analyzer instance
gpi = gp.Main(ncores = args.threads[0])

# Steps to be skipped
dskip = {
	'inst' : 1,
	'seg' : 2,
	'an' : 3,
	'box' : 3.5,
	'plot' : 4,
	'report' : 5
}
if not None is args.skip:
	gpi.skip = [dskip[e] for e in args.skip]

# Channel names
gpi.sig_names = tuple(args.sig_channels)
gpi.dna_names = tuple(args.dna_channels)

# Data directory
gpi.basedir = args.inDir[0]

# Output directory
gpi.outdir = args.outDir[0]

# Segmentation type for nuclear identification
dseg = {
	'sum_proj' : 0,
	'max_proj' : 1,
	'3d' : 2
}
gpi.seg_type = dseg[args.seg_type[0]]

# Single-nucleus analysis type
dan = {
	'sum_proj' : 0,
	'max_proj' : 1,
	'3d' : 2,
	'mid' : 3
}
gpi.an_type = dan[args.an_type[0]]

# Middle-section selection method
dmid = {
'central' : 0,
'largest' : 1,
'maxIsum' : 2
}
gpi.mid_type = dmid[args.mid_type[0]]

# Voxel aspect proportions (or sizes, ZYX)
gpi.aspect = tuple(args.aspect)

# Minimum percentage of stack to be occupied by a cell
gpi.min_z_size = args.min_z[0]

# Nuclear selection
dnsel = {
	'size' : 0,
	'surf' : 1,
	'shape' : 2,
	'sumI' : 3,
	'meanI' : 4,
	'flat_size' : 5
}
gpi.nsf = tuple([dnsel[e] for e in args.nuclear_sel])

# Regular expression to identify image files
gpi.reg = args.regexp[0]

# Where to save the run log
if None is args.logpath:
	gpi.logpath = args.outDir[0] + '/' + gpi.gen_log_name()
else:
	gpi.logpath = args.logpath[0]

# Perform deconvolved image rescaling?
gpi.rescale_deconvolved = args.rescale_deconvolved

# Normalize distance?
gpi.normalize_distance = args.normalize_distance

# Better condition naming
if not None is args.description:
	for descr in args.description:
		c, d = descr.split(':')
		gpi.cdescr[c] = d
readable_cdescr = [str(k) + ' => ' + str(v) for (k,v) in gpi.cdescr.items()]
if 0 == len(readable_cdescr):
	readable_cdescr = ["*NONE*"]

# Notes
if not None is args.note:
	gpi.notes = args.note[0]

# Debugging mode
gpi.debugging = args.DEBUG_MODE

# Show current settings
os.system('clear')
print("""
---------- SETTING:  VALUE ----------

   Input directory:  """+gpi.basedir+"""
  Output directory:  """+gpi.outdir+"""
          Log file:  """+gpi.logpath+"""

     Skipped steps:  """+str(args.skip)+"""

      DNA channels:  """+str(gpi.dna_names)+"""
   Signal channels:  """+str(gpi.sig_names)+"""

      Segmentation:  """+args.seg_type[0]+"""
          Analysis:  """+args.an_type[0]+"""
    Middle section:  """+args.mid_type[0]+"""

Voxel aspect (ZYX):  """+str(gpi.aspect)+"""
 Minimum Z portion:  """+str(gpi.min_z_size)+"""

  Condition descr.:  """+"\n                     ".join(readable_cdescr)+"""

           Threads:  """+str(gpi.ncores)+"""
              Note:  """+gpi.notes+"""

            Regexp: """+gpi.reg+"""

   Rescale deconv.:  """+str(gpi.rescale_deconvolved)+"""
   Normalize dist.:  """+str(gpi.normalize_distance)+"""
         Debug mod:  """+str(gpi.debugging)+"""

""")

# Ask for confirmation
ask('Confirm settings and proceed.')

# Start the analysis
gpi = gpi.run()

# End --------------------------------------------------------------------------

################################################################################
<|endoftext|>


<commit_before>from __future__ import division, absolute_import, print_function, unicode_literals

import os
import io
import csv

from codecs import iterdecode
from collections import namedtuple
from zipfile import ZipFile

import six

def _row_stripper(row):
    return (cell.strip() for cell in row)

class CSV(object):
    """A CSV file."""

    def __init__(self, rows, feedtype='CSVTuple'):
        self.header = list(six.next(rows))
        #deal with annoying unnecessary boms on utf-8
        self.header[0] = self.header[0].lstrip(u"\ufeff")
        self.Tuple = namedtuple(feedtype, self.header)
        self.rows = rows

    def __repr__(self):
        return '<CSV %s>' % self.header

    def __iter__(self):
        return self

    def __next__(self):
        n = six.next(self.rows)
        try:
          return self.Tuple._make(n)
        except TypeError:
          pass
    next = __next__  # python 2 compatible

class Feed(object):
    """A collection of CSV files with headers, either zipped into an archive
    or loose in a folder."""

    def __init__(self, filename, strip_fields=True):
        self.filename = filename
        self.feed_name = derive_feed_name(filename)
        self.zf = None
        self.strip_fields = strip_fields
        if not os.path.isdir(filename):
            self.zf = ZipFile(filename)
        if six.PY2:
            self.reader = self.python2_reader
        else:
            self.reader = self.python3_reader

    def __repr__(self):
        return '<Feed %s>' % self.filename

    def python2_reader(self, filename):
        if self.zf:
            try:
                binary_file_handle = self.zf.open(filename, 'rU')
            except IOError:
                raise IOError('%s is not present in feed' % filename)
        else:
            binary_file_handle = open(os.path.join(self.filename, filename), "rb")
        reader = csv.reader(binary_file_handle)
        for row in reader:
            yield [six.text_type(x, 'utf-8') for x in row]

    def python3_reader(self, filename):
        if self.zf:
            try:
                text_file_handle = io.TextIOWrapper(self.zf.open(filename, "r"), encoding="utf-8")
            except IOError:
                raise IOError('%s is not present in feed' % filename)
        else:
            text_file_handle = open(os.path.join(self.filename, filename), "r", encoding="utf-8")
        return csv.reader(text_file_handle)

    def read_table(self, filename):
        if self.strip_fields:
            rows = (_row_stripper(row) for row in self.reader(filename))
        else:
            rows = self.reader(filename)
        feedtype = filename.rsplit('/')[-1].rsplit('.')[0].title().replace('_', '')
        return CSV(feedtype=feedtype, rows=rows)

def derive_feed_name(filename):
    return os.path.basename(filename.rstrip('/'))
<commit_msg>Explicitly check for empty row.<commit_after>from __future__ import division, absolute_import, print_function, unicode_literals

import os
import io
import csv

from codecs import iterdecode
from collections import namedtuple
from zipfile import ZipFile

import six

def _row_stripper(row):
    return (cell.strip() for cell in row)

class CSV(object):
    """A CSV file."""

    def __init__(self, rows, feedtype='CSVTuple'):
        self.header = list(six.next(rows))
        #deal with annoying unnecessary boms on utf-8
        self.header[0] = self.header[0].lstrip(u"\ufeff")
        self.Tuple = namedtuple(feedtype, self.header)
        self.rows = rows

    def __repr__(self):
        return '<CSV %s>' % self.header

    def __iter__(self):
        return self

    def __next__(self):
        n = tuple(six.next(self.rows))
        if n:
          return self.Tuple._make(n)
    next = __next__  # python 2 compatible

class Feed(object):
    """A collection of CSV files with headers, either zipped into an archive
    or loose in a folder."""

    def __init__(self, filename, strip_fields=True):
        self.filename = filename
        self.feed_name = derive_feed_name(filename)
        self.zf = None
        self.strip_fields = strip_fields
        if not os.path.isdir(filename):
            self.zf = ZipFile(filename)
        if six.PY2:
            self.reader = self.python2_reader
        else:
            self.reader = self.python3_reader

    def __repr__(self):
        return '<Feed %s>' % self.filename

    def python2_reader(self, filename):
        if self.zf:
            try:
                binary_file_handle = self.zf.open(filename, 'rU')
            except IOError:
                raise IOError('%s is not present in feed' % filename)
        else:
            binary_file_handle = open(os.path.join(self.filename, filename), "rb")
        reader = csv.reader(binary_file_handle)
        for row in reader:
            yield [six.text_type(x, 'utf-8') for x in row]

    def python3_reader(self, filename):
        if self.zf:
            try:
                text_file_handle = io.TextIOWrapper(self.zf.open(filename, "r"), encoding="utf-8")
            except IOError:
                raise IOError('%s is not present in feed' % filename)
        else:
            text_file_handle = open(os.path.join(self.filename, filename), "r", encoding="utf-8")
        return csv.reader(text_file_handle)

    def read_table(self, filename):
        if self.strip_fields:
            rows = (_row_stripper(row) for row in self.reader(filename))
        else:
            rows = self.reader(filename)
        feedtype = filename.rsplit('/')[-1].rsplit('.')[0].title().replace('_', '')
        return CSV(feedtype=feedtype, rows=rows)

def derive_feed_name(filename):
    return os.path.basename(filename.rstrip('/'))
<|endoftext|>


<commit_before>#! /bin/env python

"""
>>> a = CREATED
>>> print a
Created
>>> b = INITIALIZING
>>> print b
Initializing
>>> c = INITIALIZED
>>> print c
Initialized
>>> (a>b, a==b, a<b)
(False, False, True)
>>> (a>=a, a==a, a<=a)
(True, True, True)
"""

class Status (object):
    def __init__ (self, string, val):
        self.string = string
        self.val = val
    def __str__ (self):
        return self.string
    def __gt__ (self, other):
        return self.val>other.val
    def __eq__ (self, other):
        return self.val==other.val
    def __lt__ (self, other):
        return self.val<other.val

class StatusConst (Status):
    def __init__ (self):
        pass

class StatusCreated (StatusConst):
    (string, val) = ('Created', 0)
class StatusInitializing (StatusConst):
    (string, val) = ('Initializing', 10)
class StatusInitialized (StatusConst):
    (string, val) = ('Initialized', 20)
class StatusUpdating (StatusConst):
    (string, val) = ('Updating', 30)
class StatusUpdated (StatusConst):
    (string, val) = ('Updated', 40)
class StatusFinalizing (StatusConst):
    (string, val) = ('Finalizing', 50)
class StatusFinalized (StatusConst):
    (string, val) = ('Finalized', 60)

CREATED = StatusCreated ()
INITIALIZING = StatusInitializing ()
INITIALIZED = StatusInitialized ()
UPDATING = StatusUpdating ()
UPDATED = StatusUpdated ()
FINALIZING = StatusFinalizing ()
FINALIZED = StatusFinalized ()

if __name__ == "__main__":
    import doctest
    doctest.testmod()

<commit_msg>Add __le__, __ge__ to Status class.<commit_after>#! /bin/env python
"""
>>> from __future__ import print_function
>>> a = CREATED
>>> print(a)
Created
>>> b = INITIALIZING
>>> print(b)
Initializing
>>> c = INITIALIZED
>>> print(c)
Initialized
>>> (a>b, a==b, a<b)
(False, False, True)
>>> (a>=a, a==a, a<=a)
(True, True, True)
"""

class Status(object):
    def __init__(self, string, val):
        self.string = string
        self.val = val
    def __str__(self):
        return self.string
    def __gt__(self, other):
        return self.val > other.val
    def __ge__(self, other):
        return self.val >= other.val
    def __eq__(self, other):
        return self.val == other.val
    def __lt__(self, other):
        return self.val < other.val
    def __le__(self, other):
        return self.val <= other.val

class StatusConst(Status):
    def __init__(self):
        pass

class StatusCreated(StatusConst):
    (string, val) = ('Created', 0)

class StatusInitializing(StatusConst):
    (string, val) = ('Initializing', 10)

class StatusInitialized(StatusConst):
    (string, val) = ('Initialized', 20)

class StatusUpdating(StatusConst):
    (string, val) = ('Updating', 30)

class StatusUpdated(StatusConst):
    (string, val) = ('Updated', 40)

class StatusFinalizing(StatusConst):
    (string, val) = ('Finalizing', 50)

class StatusFinalized(StatusConst):
    (string, val) = ('Finalized', 60)

CREATED = StatusCreated ()
INITIALIZING = StatusInitializing ()
INITIALIZED = StatusInitialized ()
UPDATING = StatusUpdating ()
UPDATED = StatusUpdated ()
FINALIZING = StatusFinalizing ()
FINALIZED = StatusFinalized ()

if __name__ == "__main__":
    import doctest
    doctest.testmod()

<|endoftext|>


<commit_before>"""
The Queue module.  This module handles the execution of time consuming tasks.
"""
from __future__ import absolute_import, with_statement

from subprocess import Popen, PIPE
from Queue import Queue

#: The number of CPU's available on this system
ncpu = int(Popen(['sysctl', '-n', 'hw.ncpu'], stdout=PIPE).communicate()[0])

class WorkerQueue(Queue):
  """
     The WorkerQueue class.  This class manages a pool of worker threads for
     running jobs.
  """

  _WORKER = 0
  _JOB    = 1

  def __init__(self, name, workers=1):
    """
       Initialise a worker thread pool

       @param name: The name of this thread (used in logging)
       @type name: C{str}
       @param number: The number of workers to allocate
       @type number: C{int}
    """
    Queue.__init__(self)
    from logging import getLogger
    from threading import Condition, Lock
    # We have to use our own locks since we cannot access Queue's functions
    # without not holding the locks and doing this will cause a dead lock...
    self._lock = Condition(Lock())  #: The notifier and locker of this queue
    self._log = getLogger("pypkg.queue." + name)  #: Logger of this queue
    #self._name = name  #: The name of this queue
    self._workers = workers  #: The (maximum) number of workers
    self._stalled = 0  #: The number of stalled workers

    self._stats = [1, 1]  #: Various statists (e.g. worker and job count)

    self._pool = {}  #: The pool of workers

  def __len__(self):
    """
       The size of the worker pool

       @return: The worker pool size
       @rtype: C{str}
    """
    return len(self._pool) - self._stalled

  def jid(self):
    """
       Returns the current job ID.  Can only be called from a worker thread.

       @return: The current workers JID
       @rtype: C{int}
    """
    from threading import currentThread

    return self._pool[currentThread()][1]

  def job(self, jid):
    """
       Returns if the specified job has finished or not

       @param jid: The job ID (as returns by put)
       @type jid: C{int}
       @return: If the job has finished
       @rtype: C{bool}
    """
    with self._lock:
      if jid in self._pool:
        return False
      for i in self.queue:
        if i[0] == jid:
          return False

      return True

  def pool(self):
    """
       The number of workers in the pool.  The actual number may vary but will
       stabalise to this number under full load

       @return: Number of workers
       @rtype: C{int}
    """
    return self._workers

  def set_pool(self, workers):
    """
       Changes the number of workers in the pool.  If more workers are currently
       running then some workers will be stopped after finishing their current
       job.

       @param workers: Number of workers
       @type workers: C{int}
    """
    with self._lock:
      self._workers = workers

  def put(self, func, block=True, timeout=0):
    """
       Places a job onto the queue, if insufficient workers are available one
       will be started.

       @param func: The job to execute
       @type func: C{callable}
    """
    assert callable(func)
    with self._lock:
      # If there are no workers expected then we are not open for jobs
      if not self._workers:
        return -1
      jid = self._stats[WorkerQueue._JOB]
      self._stats[WorkerQueue._JOB] += 1
      Queue.put(self, (jid, func), block, timeout)
      self._start()
      return jid

  def stats(self):
    """
       Returns a tuple about activity on the queue.
       (Workers running, Workers created, Jobs run(ning))

       @return: The tuple of information
       @rtype: C{(int, int, int)}
    """
    return (len(self) - self._stalled) + tuple(self._stats)

  def stalled(self):
    """
       Indicates if a worker has stalled.  This will result in a worker being
       created to replace this worker.

       NOTE: Can only be called from within a job
    """
    from threading import currentThread

    wid, jid = self._pool[currentThread()]

    with self._lock:
      self._log.debug("Worker %d: Job %d stalled")
      self._stalled += 1
      self._start()

      # Wait for a worker to finish
      self._lock.wait()
      self._log.debug("Worker %d: Job %d resuming")
      # `self._stalled -= 1` called by waker

  def terminate(self):
    """
       Shutdown this WorkerQueue.  Unlike set_pool(0) all remaining queued
       items are also removed.
    """
    from Queue import Empty

    self.set_pool(0)
    try:
      while True:
        self.get(False)
        self.task_done()
    except Empty:
      return

  def _start(self):
    """
       Starts a worker if there is a need.

       NOTE: Must be called with lock held
    """
    if self.qsize() > 0 and (len(self._pool) - self._stalled) < self._workers:
      from threading import Thread
      thread = Thread(target=self._worker)
      self._pool[thread] = (0, -1)
      #thread.setDaemon(True)
      thread.start()

  def _worker(self):
    """
       The worker.  It waits for a job from the queue and then executes the
       given command (with given parameters).
    """
    from threading import currentThread
    from Queue import Empty

    thread = currentThread()

    with self._lock:
      wid = self._stats[WorkerQueue._WORKER]
      self._stats[WorkerQueue._WORKER] += 1
    self._log.debug("Worker %d: Created" % wid)

    while True:
      with self._lock:
        try:
          if self._workers < (len(self._pool) - self._stalled):
            raise Empty
          jid, func = self.get(False)
          self._pool[thread] = (wid, jid)
        except Empty:
          self._pool.pop(thread)
          self._log.debug("Worker %d: Terminating" % wid)
          return

      self._work(func, jid, wid)

      self.task_done()

      # Signal that a job has finished
      with self._lock:
        self._pool[thread] = (wid, -1)
        if self._stalled:
          self._stalled -= 1
          self._lock.notify()

  def _work(self, func, jid, wid):
    """
       Execute a job

       @param func: The job to run
       @type func: C{callable}
       @param jid: The job's ID
       @type jid: C{int}
       @param wid: The worker's ID
       @type wid: C{int}
    """
    self._log.debug("Worker %d: Starting job %d" % (wid, jid))

    try:
      func()
    except KeyboardInterrupt:
      from .exit import terminate
      terminate()
    except BaseException:
      self._log.exception("Worker %d: Job %d threw an exception" % (wid, jid))
    finally:
      self._log.debug("Worker %d: Finished job %d" % (wid, jid))

config_queue  = WorkerQueue("config", 1)  #: Queue for configuring port options
build_queue   = WorkerQueue("build", ncpu)  #: Queue for building ports
fetch_queue   = WorkerQueue("fetch", 1)  #: Queue for fetching dist files
install_queue = WorkerQueue("install", 1)  #: Queue for installing ports
ports_queue   = WorkerQueue("ports", ncpu * 2)  #: Queue for fetching port info
queues        = [config_queue, build_queue, fetch_queue, install_queue,
                 ports_queue]  #: List of all the queues
<commit_msg>- Fix checking for JID status<commit_after>"""
The Queue module.  This module handles the execution of time consuming tasks.
"""
from __future__ import absolute_import, with_statement

from subprocess import Popen, PIPE
from Queue import Queue

#: The number of CPU's available on this system
ncpu = int(Popen(['sysctl', '-n', 'hw.ncpu'], stdout=PIPE).communicate()[0])

class WorkerQueue(Queue):
  """
     The WorkerQueue class.  This class manages a pool of worker threads for
     running jobs.
  """

  _WORKER = 0
  _JOB    = 1

  def __init__(self, name, workers=1):
    """
       Initialise a worker thread pool

       @param name: The name of this thread (used in logging)
       @type name: C{str}
       @param number: The number of workers to allocate
       @type number: C{int}
    """
    Queue.__init__(self)
    from logging import getLogger
    from threading import Condition, Lock
    # We have to use our own locks since we cannot access Queue's functions
    # without not holding the locks and doing this will cause a dead lock...
    self._lock = Condition(Lock())  #: The notifier and locker of this queue
    self._log = getLogger("pypkg.queue." + name)  #: Logger of this queue
    #self._name = name  #: The name of this queue
    self._workers = workers  #: The (maximum) number of workers
    self._stalled = 0  #: The number of stalled workers

    self._stats = [1, 1]  #: Various statists (e.g. worker and job count)

    self._pool = {}  #: The pool of workers

  def __len__(self):
    """
       The size of the worker pool

       @return: The worker pool size
       @rtype: C{str}
    """
    return len(self._pool) - self._stalled

  def jid(self):
    """
       Returns the current job ID.  Can only be called from a worker thread.

       @return: The current workers JID
       @rtype: C{int}
    """
    from threading import currentThread

    return self._pool[currentThread()][1]

  def job(self, jid):
    """
       Returns if the specified job has finished or not

       @param jid: The job ID (as returns by put)
       @type jid: C{int}
       @return: If the job has finished
       @rtype: C{bool}
    """
    with self._lock:
      for i in self._pool:
        if i[1] == jid:
          return False
      for i in self.queue:
        if i[0] == jid:
          return False

      return True

  def pool(self):
    """
       The number of workers in the pool.  The actual number may vary but will
       stabalise to this number under full load

       @return: Number of workers
       @rtype: C{int}
    """
    return self._workers

  def set_pool(self, workers):
    """
       Changes the number of workers in the pool.  If more workers are currently
       running then some workers will be stopped after finishing their current
       job.

       @param workers: Number of workers
       @type workers: C{int}
    """
    with self._lock:
      self._workers = workers

  def put(self, func, block=True, timeout=0):
    """
       Places a job onto the queue, if insufficient workers are available one
       will be started.

       @param func: The job to execute
       @type func: C{callable}
    """
    assert callable(func)
    with self._lock:
      # If there are no workers expected then we are not open for jobs
      if not self._workers:
        return -1
      jid = self._stats[WorkerQueue._JOB]
      self._stats[WorkerQueue._JOB] += 1
      Queue.put(self, (jid, func), block, timeout)
      self._start()
      return jid

  def stats(self):
    """
       Returns a tuple about activity on the queue.
       (Workers running, Workers created, Jobs run(ning))

       @return: The tuple of information
       @rtype: C{(int, int, int)}
    """
    return (len(self) - self._stalled) + tuple(self._stats)

  def stalled(self):
    """
       Indicates if a worker has stalled.  This will result in a worker being
       created to replace this worker.

       NOTE: Can only be called from within a job
    """
    from threading import currentThread

    wid, jid = self._pool[currentThread()]

    with self._lock:
      self._log.debug("Worker %d: Job %d stalled")
      self._stalled += 1
      self._start()

      # Wait for a worker to finish
      self._lock.wait()
      self._log.debug("Worker %d: Job %d resuming")
      # `self._stalled -= 1` called by waker

  def terminate(self):
    """
       Shutdown this WorkerQueue.  Unlike set_pool(0) all remaining queued
       items are also removed.
    """
    from Queue import Empty

    self.set_pool(0)
    try:
      while True:
        self.get(False)
        self.task_done()
    except Empty:
      return

  def _start(self):
    """
       Starts a worker if there is a need.

       NOTE: Must be called with lock held
    """
    if self.qsize() > 0 and (len(self._pool) - self._stalled) < self._workers:
      from threading import Thread
      thread = Thread(target=self._worker)
      self._pool[thread] = (0, -1)
      #thread.setDaemon(True)
      thread.start()

  def _worker(self):
    """
       The worker.  It waits for a job from the queue and then executes the
       given command (with given parameters).
    """
    from threading import currentThread
    from Queue import Empty

    thread = currentThread()

    with self._lock:
      wid = self._stats[WorkerQueue._WORKER]
      self._stats[WorkerQueue._WORKER] += 1
    self._log.debug("Worker %d: Created" % wid)

    while True:
      with self._lock:
        try:
          if self._workers < (len(self._pool) - self._stalled):
            raise Empty
          jid, func = self.get(False)
          self._pool[thread] = (wid, jid)
        except Empty:
          self._pool.pop(thread)
          self._log.debug("Worker %d: Terminating" % wid)
          return

      self._work(func, jid, wid)

      self.task_done()

      # Signal that a job has finished
      with self._lock:
        self._pool[thread] = (wid, -1)
        if self._stalled:
          self._stalled -= 1
          self._lock.notify()

  def _work(self, func, jid, wid):
    """
       Execute a job

       @param func: The job to run
       @type func: C{callable}
       @param jid: The job's ID
       @type jid: C{int}
       @param wid: The worker's ID
       @type wid: C{int}
    """
    self._log.debug("Worker %d: Starting job %d" % (wid, jid))

    try:
      func()
    except KeyboardInterrupt:
      from .exit import terminate
      terminate()
    except BaseException:
      self._log.exception("Worker %d: Job %d threw an exception" % (wid, jid))
    finally:
      self._log.debug("Worker %d: Finished job %d" % (wid, jid))

config_queue  = WorkerQueue("config", 1)  #: Queue for configuring port options
build_queue   = WorkerQueue("build", ncpu)  #: Queue for building ports
fetch_queue   = WorkerQueue("fetch", 1)  #: Queue for fetching dist files
install_queue = WorkerQueue("install", 1)  #: Queue for installing ports
ports_queue   = WorkerQueue("ports", ncpu * 2)  #: Queue for fetching port info
queues        = [config_queue, build_queue, fetch_queue, install_queue,
                 ports_queue]  #: List of all the queues
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding=utf-8
"""
Extract module

"""
from __future__ import unicode_literals
from __future__ import absolute_import

__author__ = "Clare Corthell"
__copyright__ = "Copyright 2015, summer.ai"
__date__ = "2015-11-20"
__email__ = "clare@summer.ai"

import requests
from lxml import etree
from .config import config  # Make sure to use absolute imports here
import html2text
from serapis.util import squashed
import re
import logging
from nltk.tokenize import sent_tokenize

log = logging.getLogger('serapis.extract')
html_parser = html2text.HTML2Text()
html_parser.ignore_links = html_parser.ignore_images = html_parser.ignore_emphasis = True
html_parser.body_width = 0

class PageRequest(object):
    """
    Request for single page

    initialize with param
    `url` and `term`

    returns attributes
    .response
    .structured

        'url'
        'html'
        'text'
        'title'
        'author'
        'date'

    """

    OPENING_QUOTE = '|'.join(("\"", "'", """, "“", "“", "‘", "‘", "«", "«", "‹", "‹", "„", "„", "‚", "‚"))
    CLOSING_QUOTE = '|'.join(("'", """, "”", "”", "’", "’", "»", "»", "›", "›", "“", "“", "‘", "‘"))

    def request_page(self):
        """
        Returns a utf-8 encoded string

        """
        try:
            self.response = requests.get(self.url, timeout=3)
            return self.response
        except:
            log.error("Failed to return page for url: %s" % self.url)
            return None

    def get_sanitized_page_text(self, page_text):
        """
        Returns page text as

        """
        sentences = []
        matches = []
        for paragraph in self.text.split('\n\n'):
            if len(paragraph) > 30:
                for sentence in sent_tokenize(paragraph):
                    sentences.append(sentence)
                    if squashed(self.term) in squashed(sentence):
                        matches.append({
                            'term': self.term,
                            'sentence': sentence.strip(" *#")
                        })
        return sentences

    def get_meta(self):
        """
        Scan meta tags for keys: 'title', 'author', 'date'

        If 'parsely-page' attribute exists, use those values

        Currently arbitrarily returns the first found value for each key

        TODO
        - Metadata is sometimes in XML <PageMap> object

        """
        # Common ways to encode authorship information
        prop_names = {
            "name": "author",  # W3C
            "property": "og:author",  # Facebook OG
            "property": "article:author",  # Pinterest and Twitter meta
        }
        authors = []

        tree = etree.HTML(self.response.text)  # TODO may change this to response.content, access as bytes
        for prop, value in prop_names.items():
            tags = tree.xpath("//meta[@{}='{}']".format(prop, value))
            authors.extend([tag.attrib.get('content') for tag in tags])

        # Pick the first author that looks reasonable
        authors = filter(lambda author: author and not author.startswith("http"), authors)
        author = authors[0] if authors else None

        title_tags = tree.xpath("//title")
        title = title_tags[0].text if title_tags else None

        return {
            'author': author,
            'title': title
        }

    def get_html_features(self, html):
        """Detects whether the search term exists is highlighted (bolded, emphasised) or
        in quotes. Needs to be called after self.request_page.

        Returns:
            dict -- dict of bools for different features.
        """
        if not self.term:
            return None

        minimal_html = squashed(self.html, keep='<>/&;')
        minimal_term = squashed(self.term)

        highlight_re = r"<(em|i|b|strong|span)[^>]*> *{}[ ,:]*</\1>".format(minimal_term)
        quote_re = r"<({})[^>]*> *{}[ ,:]*</({})>".format(self.OPENING_QUOTE, minimal_term, self.CLOSING_QUOTE)

        features = {
            "highlighted": bool(re.search(highlight_re, minimal_html, re.IGNORECASE)),
            "quotes": bool(re.search(quote_re, minimal_html, re.IGNORECASE)),
        }
        return features

    def get_structured_page(self):
        """
        Returns elements extracted from html

        """
        if not self.response:
            return None

        html = self.response.text
        self.text = self.get_sanitized_page_text(html_parser.handle(html))
        self.features = self.get_html_features(html)

        self.structured = {
            "term": self.term,
            "url": self.url,
            "doc": self.text,
            "features": self.features,
            "matches": self.matches
        }

        self.structured.update(self.get_meta())

        if config.save_html:
            self.structured["html"] = html

        return self.structured

    def __init__(self, url, term):
        self.url = url
        self.term = term
        self.response = self.request_page()
        self.structured = self.get_structured_page()

class DiffbotRequest:
    """
    Request for single page from Diffbot

    takes
    `url`

    Diffbot API returns json object

    """
    def get_article(self):
        self.params['url'] = self.url
        self.response = requests.get(self.diff_article_api, params=self.params).json()
        if not self.response.get('objects'):
            if self.response.get('error'):
                print("Response Error '{}' (code: {})".format(self.response['error'], self.response['errorCode']))
            else:
                print("NO RESULTS")
                raise Exception

        return self.response

    def get_structured_response(self):
        self.params['url'] = self.url
        self.response = self.get_article(self.url)

        results = list()
        for object in self.response.get('objects', []):
            if object.get('text'):
                result = {
                    "title": object.get('title'),
                    "url": object.get('pageUrl'),
                    "author": object.get('author'),
                    "html": object.get('html'),
                    "tags": object.get('tags'),
                    "doc": object.get('text'),
                    # "date": parse_date(object.get('date', '')).isoformat(),
                }
                results.append(result)
        self.structured = results
        return results

    def __init__(self, url):
        self.url = url
        self.response = None
        self.structured = None
        self.diff_article_api = 'http://api.diffbot.com/v3/article'
        self.params = {
            'token': config.credentials['diffbot'],
            'url': self.url
        }
<commit_msg>bug fix self refs<commit_after>#!/usr/bin/env python
# coding=utf-8
"""
Extract module

"""
from __future__ import unicode_literals
from __future__ import absolute_import

__author__ = "Clare Corthell"
__copyright__ = "Copyright 2015, summer.ai"
__date__ = "2015-11-20"
__email__ = "clare@summer.ai"

import requests
from lxml import etree
from .config import config  # Make sure to use absolute imports here
import html2text
from serapis.util import squashed
import re
import logging
from nltk.tokenize import sent_tokenize

log = logging.getLogger('serapis.extract')
html_parser = html2text.HTML2Text()
html_parser.ignore_links = html_parser.ignore_images = html_parser.ignore_emphasis = True
html_parser.body_width = 0

class PageRequest(object):
    """
    Request for single page

    initialize with param
    `url` and `term`

    returns attributes
    .response
    .structured

        'url'
        'html'
        'text'
        'title'
        'author'
        'date'

    """

    OPENING_QUOTE = '|'.join(("\"", "'", """, "“", "“", "‘", "‘", "«", "«", "‹", "‹", "„", "„", "‚", "‚"))
    CLOSING_QUOTE = '|'.join(("'", """, "”", "”", "’", "’", "»", "»", "›", "›", "“", "“", "‘", "‘"))

    def request_page(self):
        """
        Returns a utf-8 encoded string

        """
        try:
            self.response = requests.get(self.url, timeout=3)
            return self.response
        except:
            log.error("Failed to return page for url: %s" % self.url)
            return None

    def get_sanitized_page_text(self, page_text):
        """
        Returns page text as

        """
        sentences = []
        self.matches = []
        for paragraph in page_text.split('\n\n'):
            if len(paragraph) > 30:
                for sentence in sent_tokenize(paragraph):
                    sentences.append(sentence)
                    if self.term and squashed(self.term) in squashed(sentence):
                        self.matches.append({
                            'term': self.term,
                            'sentence': sentence.strip(" *#")
                        })
        return sentences

    def get_meta(self):
        """
        Scan meta tags for keys: 'title', 'author', 'date'

        If 'parsely-page' attribute exists, use those values

        Currently arbitrarily returns the first found value for each key

        TODO
        - Metadata is sometimes in XML <PageMap> object

        """
        # Common ways to encode authorship information
        prop_names = {
            "name": "author",  # W3C
            "property": "og:author",  # Facebook OG
            "property": "article:author",  # Pinterest and Twitter meta
        }
        authors = []

        tree = etree.HTML(self.response.text)  # TODO may change this to response.content, access as bytes
        for prop, value in prop_names.items():
            tags = tree.xpath("//meta[@{}='{}']".format(prop, value))
            authors.extend([tag.attrib.get('content') for tag in tags])

        # Pick the first author that looks reasonable
        authors = filter(lambda author: author and not author.startswith("http"), authors)
        author = authors[0] if authors else None

        title_tags = tree.xpath("//title")
        title = title_tags[0].text if title_tags else None

        return {
            'author': author,
            'title': title
        }

    def get_html_features(self, html):
        """Detects whether the search term exists is highlighted (bolded, emphasised) or
        in quotes. Needs to be called after self.request_page.

        Returns:
            dict -- dict of bools for different features.
        """
        if not self.term:
            return None

        minimal_html = squashed(self.html, keep='<>/&;')
        minimal_term = squashed(self.term)

        highlight_re = r"<(em|i|b|strong|span)[^>]*> *{}[ ,:]*</\1>".format(minimal_term)
        quote_re = r"<({})[^>]*> *{}[ ,:]*</({})>".format(self.OPENING_QUOTE, minimal_term, self.CLOSING_QUOTE)

        features = {
            "highlighted": bool(re.search(highlight_re, minimal_html, re.IGNORECASE)),
            "quotes": bool(re.search(quote_re, minimal_html, re.IGNORECASE)),
        }
        return features

    def get_structured_page(self):
        """
        Returns elements extracted from html

        """
        if not self.response:
            return None

        html = self.response.text
        self.text = self.get_sanitized_page_text(html_parser.handle(html))
        self.features = self.get_html_features(html)

        self.structured = {
            "term": self.term,
            "url": self.url,
            "doc": self.text,
            "features": self.features,
            "matches": self.matches
        }

        self.structured.update(self.get_meta())

        if config.save_html:
            self.structured["html"] = html

        return self.structured

    def __init__(self, url, term):
        self.url = url
        self.term = term
        self.response = self.request_page()
        self.structured = self.get_structured_page()

class DiffbotRequest:
    """
    Request for single page from Diffbot

    takes
    `url`

    Diffbot API returns json object

    """
    def get_article(self):
        self.params['url'] = self.url
        self.response = requests.get(self.diff_article_api, params=self.params).json()
        if not self.response.get('objects'):
            if self.response.get('error'):
                print("Response Error '{}' (code: {})".format(self.response['error'], self.response['errorCode']))
            else:
                print("NO RESULTS")
                raise Exception

        return self.response

    def get_structured_response(self):
        self.params['url'] = self.url
        self.response = self.get_article(self.url)

        results = list()
        for object in self.response.get('objects', []):
            if object.get('text'):
                result = {
                    "title": object.get('title'),
                    "url": object.get('pageUrl'),
                    "author": object.get('author'),
                    "html": object.get('html'),
                    "tags": object.get('tags'),
                    "doc": object.get('text'),
                    # "date": parse_date(object.get('date', '')).isoformat(),
                }
                results.append(result)
        self.structured = results
        return results

    def __init__(self, url):
        self.url = url
        self.response = None
        self.structured = None
        self.diff_article_api = 'http://api.diffbot.com/v3/article'
        self.params = {
            'token': config.credentials['diffbot'],
            'url': self.url
        }
<|endoftext|>


<commit_before>import os
import json
from lxml import etree

from girder.api.rest import Resource, loadmodel, boundHandler
from girder.api import access
from girder.api.describe import Description, describeRoute
from girder.constants import AccessType
from girder.plugins.worker import utils as wutils

def genHandlerToRunCLI(restResource, xmlFile, scriptFile):
    """Generates a handler to run CLI using girder_worker

    Parameters
    ----------
    restResource : girder.api.rest.Resource
        The object of a class derived from girder.api.rest.Resource to which
        this handler will be attached
    xmlFile : str
        Full path to xml file of the CLI
    scriptFile : str
        Full path to .py file containing the code of the clu

    Returns
    -------
    function
        Returns a function that runs the CLI using girder_worker
    """

    xmlPath, xmlNameWExt = os.path.split(xmlFile)
    xmlName = os.path.splitext(xmlNameWExt)[0]

    # parse xml of cli
    clixml = etree.parse(xmlFile)

    # read the script file containing the code into a string
    with open(scriptFile) as f:
        codeToRun = f.read()

    # do stuff needed to create REST endpoint for cLI
    handlerDesc = Description(clixml.findtext('title'))
    taskSpec = {'name': xmlName,
                'mode': 'python',
                'inputs': [],
                'outputs': []}

    slicerToGirderTypeMap = {
        'boolean': 'boolean',
        'integer': 'integer',
        'float': 'number',
        'double': 'number',
        'string': 'string',
        'integer-vector': 'integer_list',
        'float-vector': 'number_list',
        'double-vector': 'number_list',
        'string-vector': 'string_list',
        'integer-enumeration': 'integer',
        'float-enumeration': 'number',
        'double-enumeration': 'number',
        'string-enumeration': 'string',
        'file': 'string',
        'directory': 'string',
        'image': 'string'}

    inputGirderSuffix = '_girderId'
    outputGirderSuffix = '_folder_girderId'
    outGirderNameSuffix = '_name'

    def getDefaultValFromString(strVal, typeVal):
        if typeVal in ['integer', 'integer-enumeration']:
            return int(strVal)
        elif typeVal in ['float', 'float-enumeration',
                         'double', 'double-enumeration']:
            return float(strVal)
        elif typeVal == 'integer-vector':
            return [int(e.strip()) for e in strVal.split(',')]
        elif typeVal in ['float-vector', 'double-vector']:
            return [float(e.strip()) for e in strVal.split(',')]
        elif typeVal == 'string-vector':
            return [str(e.strip()) for e in strVal.split(',')]
        else:
            return typeVal

    # identify xml elements of input, output, and optional params
    ioXMLElements = []
    paramXMLElements = []
    inputXMLElements = []
    outputXMLElements = []

    for pgelt in clixml.getiterator('parameters'):
        for pelt in pgelt:
            if pelt.tag in ['description', 'label']:
                continue

            if pelt.tag not in slicerToGirderTypeMap.keys():
                raise Exception(
                    'Parameter type %s is currently not supported' %
                    pelt.tag)

            channel = pelt.findtext('channel')
            if channel is not None:
                if channel.lower() not in ['input', 'output']:
                    raise Exception('channel must be input or output.')
                ioXMLElements.append(pelt)
            else:
                if pelt.tag in ['image', 'file', 'directory']:
                    raise Exception('optional parameters of type'
                                    'image, file, or directory are '
                                    'currently not supported')
                paramXMLElements.append(pelt)

    ioXMLElements = sorted(ioXMLElements,
                           key=lambda elt: elt.findtext('index'))

    for elt in ioXMLElements:
        if elt.findtext('channel').lower() == 'input':
            inputXMLElements.append(elt)
        else:
            if elt.tag not in ['image', 'file', 'directory']:
                raise Exception(
                    'outputs of type other than image, file, or '
                    'directory are not currently supported.')
            outputXMLElements.append(elt)

    # generate task spec for inputs
    for elt in inputXMLElements:
        curName = elt.findtext('name')
        curType = elt.tag
        curDesc = elt.findtext('description')

        curTaskSpec = dict()
        curTaskSpec['id'] = curName
        curTaskSpec['type'] = slicerToGirderTypeMap[curType]
        curTaskSpec['format'] = slicerToGirderTypeMap[curType]

        if curType in ['image', 'file', 'directory']:
            curTaskSpec['target'] = 'filepath'  # check
            handlerDesc.param(curName + inputGirderSuffix,
                              'Girder ID of input %s - %s: %s'
                              % (curType, curName, curDesc),
                              dataType='string')
        else:
            handlerDesc.param(curName, curDesc, dataType='string')

        taskSpec['inputs'].append(curTaskSpec)

    # generate task spec for optional parameters
    for elt in paramXMLElements:
        curName = elt.findtext('name')
        curType = elt.tag
        curDesc = elt.findtext('description')

        curTaskSpec = dict()
        curTaskSpec['id'] = curName
        curTaskSpec['type'] = slicerToGirderTypeMap[curType]
        curTaskSpec['format'] = slicerToGirderTypeMap[curType]

        defaultValSpec = dict()
        defaultValSpec['format'] = curTaskSpec['format']
        strDefaultVal = elt.findtext('default')
        if strDefaultVal is not None:
            defaultVal = getDefaultValFromString(strDefaultVal,
                                                 curType)
        elif curType == 'boolean':
            defaultVal = False
        else:
            raise Exception(
                'optional parameters of type %s must '
                'provide a default value in the xml' % curType)
        defaultValSpec['data'] = defaultVal
        curTaskSpec['default'] = defaultValSpec

        handlerDesc.param(curName,
                          curDesc,
                          dataType='string',
                          required=False,
                          default=json.dumps(defaultVal))

        taskSpec['inputs'].append(curTaskSpec)

    # generate task spec for outputs
    for elt in outputXMLElements:
        curName = elt.findtext('name')
        curType = elt.tag
        curDesc = elt.findtext('description')

        # task spec for the current output
        curTaskSpec = dict()
        curTaskSpec['id'] = curName
        curTaskSpec['type'] = slicerToGirderTypeMap[curType]
        curTaskSpec['format'] = slicerToGirderTypeMap[curType]

        if curType in ['image', 'file', 'directory']:
            curTaskSpec['target'] = 'filepath'  # check

        taskSpec['outputs'].append(curTaskSpec)

        # param for parent folder
        handlerDesc.param(curName + outputGirderSuffix,
                          'Girder ID of parent folder '
                          'for output %s - %s: %s'
                          % (curType, curName, curDesc),
                          dataType='string')

        # param for name by which to store the current output
        handlerDesc.param(curName + outGirderNameSuffix,
                          'Name of output %s - %s: %s'
                          % (curType, curName, curDesc),
                          dataType='string')

    # define CLI handler function
    @boundHandler(restResource)
    @access.user
    @describeRoute(handlerDesc)
    def cliHandler(self, **args):

        user = self.getCurrentUser()
        token = self.getCurrentToken()['_id']

        # create job
        jobModel = self.model('job', 'jobs')
        jobTitle = '.'.join((restResource.resourceName, xmlName))
        job = jobModel.createJob(title=jobTitle,
                                 type=jobTitle,
                                 handler='worker_handler',
                                 user=user)
        kwargs = {
            'validate': False,
            'auto_convert': True,
            'cleanup': True}
        taskSpec['script'] = codeToRun

        # create job info
        jobToken = jobModel.createJobToken(job)
        kwargs['jobInfo'] = wutils.jobInfoSpec(job, jobToken)

        # create input bindings
        kwargs['inputs'] = dict()
        for elt in inputXMLElements:
            curName = elt.findtext('name')
            curType = elt.tag

            curBindingSpec = dict()
            if curType in ['image', 'file', 'directory']:
                # inputs of type image, file, or directory
                # should be located on girder
                if curType in ['image', 'file']:
                    curGirderType = 'item'
                else:
                    curGirderType = 'folder'
                curBindingSpec = wutils.girderInputSpec(
                    args[curName],
                    resourceType=curGirderType,
                    dataType='string', dataFormat='string',
                    token=token)
            else:
                # inputs that are not of type image, file, or directory
                # should be passed inline as string from json.dumps()
                curBindingSpec['mode'] = 'inline'
                curBindingSpec['type'] = slicerToGirderTypeMap[curType]
                curBindingSpec['format'] = 'json'
                curBindingSpec['data'] = args['params'][curName]

            kwargs['inputs'][curName] = curBindingSpec

        # create optional parameter bindings
        for elt in paramXMLElements:
            curName = elt.findtext('name')
            curType = elt.tag

            curBindingSpec = dict()
            curBindingSpec['mode'] = 'inline'
            curBindingSpec['type'] = slicerToGirderTypeMap[curType]
            curBindingSpec['format'] = 'json'
            curBindingSpec['data'] = args['params'][curName]

            kwargs['inputs'][curName] = curBindingSpec

        # create output boundings
        kwargs['outputs'] = dict()
        for elt in outputXMLElements:
            curName = elt.findtext('name')

            curBindingSpec = wutils.girderOutputSpec(
                args[curName],
                token,
                name=args['params'][curName + outGirderNameSuffix],
                dataType='string', dataFormat='string')
            kwargs['outputs'][curName] = curBindingSpec

            codeToSetOutputPath = "%s = os.path.join(_tempdir, '%s')" % (
                curName, args['params'][curName + outGirderNameSuffix])
            taskSpec['script'] = '\n'.join((codeToSetOutputPath,
                                           taskSpec['script']))

        # create task spec
        taskSpec['script'] = '\n'.join(("import os", taskSpec['script']))
        kwargs['task'] = taskSpec

        # schedule job
        job['kwargs'] = kwargs
        job = jobModel.save(job)
        jobModel.scheduleJob(job)

        # return result
        return jobModel.filter(job, user)

    handlerFunc = cliHandler

    # loadmodel stuff for inputs in girder
    for elt in inputXMLElements:
        curName = elt.findtext('name')
        curType = elt.tag

        if curType in ['image', 'file']:
            curModel = 'item'
        elif curType == 'directory':
            curModel = 'folder'
        else:
            continue
        curMap = {curName + inputGirderSuffix: curName}

        handlerFunc = loadmodel(map=curMap,
                                model=curModel,
                                level=AccessType.READ)(handlerFunc)

    # loadmodel stuff for outputs to girder
    for elt in outputXMLElements:
        curName = elt.findtext('name')

        curModel = 'folder'
        curMap = {curName + outputGirderSuffix: curName}

        handlerFunc = loadmodel(map=curMap,
                                model=curModel,
                                level=AccessType.WRITE)(handlerFunc)

    return handlerFunc

def genHandlerToGetCLIXmlSpec(restResource, xmlFile):
    """Generates a handler that returns the XML spec of the CLI

    Parameters
    ----------
    restResource : girder.api.rest.Resource
        The object of a class derived from girder.api.rest.Resource to which
        this handler will be attached
    xmlFile : str
        Full path to xml file of the CLI

    Returns
    -------
    function
        Returns a function that returns the xml spec of the CLI
    """

    xmlPath, xmlNameWExt = os.path.split(xmlFile)
    xmlName = os.path.splitext(xmlNameWExt)[0]

    # parse xml of cli
    clixml = etree.parse(xmlFile)

    # define the handler that returns the CLI's xml spec
    @boundHandler(restResource)
    @access.user
    @describeRoute(
        Description('Get XML spec of %s CLI' % xmlName)
    )
    def getXMLSpecHandler(self, *args, **kwargs):
        return etree.tostring(clixml)

    return getXMLSpecHandler

def genRESTEndPointsForSlicerCLIsInSubDirs(info, restResourceName, cliRootDir):
    """Generates REST end points for slicer CLIs placed in subdirectories of a
    given root directory and attaches them to a REST resource with the given
    name.

    For each CLI, it creates:
    * a GET Route (<apiURL>/`restResourceName`/<cliName>/xmlspec) that returns
    the xml spec of the CLI
    * a POST Route (<apiURL>/`restResourceName`/<cliName>/run) that runs
    the CLI

    It also creates a GET route (<apiURL>/`restResourceName`) that returns a
    list of relative routes to all CLIs attached to the generated REST resource

    Parameters
    ----------
    info
    restResourceName : str
        Name of the REST resource to which the end-points should be attached
    cliRootDir : str
        Full path of root directory containing the CLIs

    Returns
    -------

    """

    # create REST resource
    restResource = type(restResourceName,
                        (Resource, ),
                        {'resourceName': restResourceName})()

    # Add REST route for slicer CLIs located in subdirectories
    subdirList = [child
                  for child in os.listdir(cliRootDir)
                  if os.path.isdir(os.path.join(cliRootDir, child))]

    cliList = []

    for parentdir, dirnames, filenames in os.walk(cliRootDir):
        for subdir in dirnames:
            curCLIRelPath = os.path.relpath(os.path.join(parentdir, subdir),
                                            cliRootDir)

            # check if subdir contains a .xml file with the same name
            xmlFile = os.path.join(parentdir, subdir, subdir + '.xml')

            if not os.path.isfile(xmlFile):
                continue

            xmlPath, xmlNameWExt = os.path.split(xmlFile)
            xmlName = os.path.splitext(xmlNameWExt)[0]

            # check if subdir contains a .py file with the same name
            scriptFile = os.path.join(xmlPath, xmlName + '.py')

            if not os.path.isfile(scriptFile):
                continue

            print curCLIRelPath

            # TODO: check if xml adheres to slicer execution model xml schema

            # create a POST REST route that runs the CLI by invoking the handler
            try:
                cliRunHandler = genHandlerToRunCLI(restResource,
                                                   xmlFile, scriptFile)
            except Exception as e:
                print "Failed to create REST endpoints for %s: %s" % (
                    curCLIRelPath, e)
                continue

            cliRunHandlerName = 'run_' + xmlName
            setattr(restResource, cliRunHandlerName, cliRunHandler)
            restResource.route('POST',
                               (curCLIRelPath, 'run'),
                               getattr(restResource, cliRunHandlerName))

            # create GET REST route that returns the xml of the CLI
            try:
                cliGetXMLSpecHandler = genHandlerToGetCLIXmlSpec(restResource,
                                                                 xmlFile)
            except Exception as e:
                print "Failed to create REST endpoints for %s: %s" % (
                    curCLIRelPath, e)
                continue

            cliGetXMLSpecHandlerName = 'get_xml_' + xmlName
            setattr(restResource,
                    cliGetXMLSpecHandlerName,
                    cliGetXMLSpecHandler)
            restResource.route('GET',
                               (curCLIRelPath, 'xmlspec',),
                               getattr(restResource, cliGetXMLSpecHandlerName))

            cliList.append(curCLIRelPath)

    # create GET route that returns a list of relative routes to all CLIs
    @boundHandler(restResource)
    @access.user
    @describeRoute(
        Description('Get list of relative routes to all CLIs')
    )
    def getCLIListHandler(self, *args, **kwargs):
        return cliList

    getCLIListHandlerName = 'get_cli_list'
    setattr(restResource, getCLIListHandlerName, getCLIListHandler)
    restResource.route('GET', (), getattr(restResource, getCLIListHandlerName))

    # expose the generated REST resource via apiRoot
    setattr(info['apiRoot'], restResourceName, restResource)

def load(info):
    cliRootDir = os.path.dirname(__file__)
    genRESTEndPointsForSlicerCLIsInSubDirs(info, 'HistomicsTK', cliRootDir)
<commit_msg>Fixed style error<commit_after>import os
import json
from lxml import etree

from girder.api.rest import Resource, loadmodel, boundHandler
from girder.api import access
from girder.api.describe import Description, describeRoute
from girder.constants import AccessType
from girder.plugins.worker import utils as wutils

def genHandlerToRunCLI(restResource, xmlFile, scriptFile):
    """Generates a handler to run CLI using girder_worker

    Parameters
    ----------
    restResource : girder.api.rest.Resource
        The object of a class derived from girder.api.rest.Resource to which
        this handler will be attached
    xmlFile : str
        Full path to xml file of the CLI
    scriptFile : str
        Full path to .py file containing the code of the clu

    Returns
    -------
    function
        Returns a function that runs the CLI using girder_worker
    """

    xmlPath, xmlNameWExt = os.path.split(xmlFile)
    xmlName = os.path.splitext(xmlNameWExt)[0]

    # parse xml of cli
    clixml = etree.parse(xmlFile)

    # read the script file containing the code into a string
    with open(scriptFile) as f:
        codeToRun = f.read()

    # do stuff needed to create REST endpoint for cLI
    handlerDesc = Description(clixml.findtext('title'))
    taskSpec = {'name': xmlName,
                'mode': 'python',
                'inputs': [],
                'outputs': []}

    slicerToGirderTypeMap = {
        'boolean': 'boolean',
        'integer': 'integer',
        'float': 'number',
        'double': 'number',
        'string': 'string',
        'integer-vector': 'integer_list',
        'float-vector': 'number_list',
        'double-vector': 'number_list',
        'string-vector': 'string_list',
        'integer-enumeration': 'integer',
        'float-enumeration': 'number',
        'double-enumeration': 'number',
        'string-enumeration': 'string',
        'file': 'string',
        'directory': 'string',
        'image': 'string'}

    inputGirderSuffix = '_girderId'
    outputGirderSuffix = '_folder_girderId'
    outGirderNameSuffix = '_name'

    def getDefaultValFromString(strVal, typeVal):
        if typeVal in ['integer', 'integer-enumeration']:
            return int(strVal)
        elif typeVal in ['float', 'float-enumeration',
                         'double', 'double-enumeration']:
            return float(strVal)
        elif typeVal == 'integer-vector':
            return [int(e.strip()) for e in strVal.split(',')]
        elif typeVal in ['float-vector', 'double-vector']:
            return [float(e.strip()) for e in strVal.split(',')]
        elif typeVal == 'string-vector':
            return [str(e.strip()) for e in strVal.split(',')]
        else:
            return typeVal

    # identify xml elements of input, output, and optional params
    ioXMLElements = []
    paramXMLElements = []
    inputXMLElements = []
    outputXMLElements = []

    for pgelt in clixml.getiterator('parameters'):
        for pelt in pgelt:
            if pelt.tag in ['description', 'label']:
                continue

            if pelt.tag not in slicerToGirderTypeMap.keys():
                raise Exception(
                    'Parameter type %s is currently not supported' %
                    pelt.tag)

            channel = pelt.findtext('channel')
            if channel is not None:
                if channel.lower() not in ['input', 'output']:
                    raise Exception('channel must be input or output.')
                ioXMLElements.append(pelt)
            else:
                if pelt.tag in ['image', 'file', 'directory']:
                    raise Exception('optional parameters of type'
                                    'image, file, or directory are '
                                    'currently not supported')
                paramXMLElements.append(pelt)

    ioXMLElements = sorted(ioXMLElements,
                           key=lambda elt: elt.findtext('index'))

    for elt in ioXMLElements:
        if elt.findtext('channel').lower() == 'input':
            inputXMLElements.append(elt)
        else:
            if elt.tag not in ['image', 'file', 'directory']:
                raise Exception(
                    'outputs of type other than image, file, or '
                    'directory are not currently supported.')
            outputXMLElements.append(elt)

    # generate task spec for inputs
    for elt in inputXMLElements:
        curName = elt.findtext('name')
        curType = elt.tag
        curDesc = elt.findtext('description')

        curTaskSpec = dict()
        curTaskSpec['id'] = curName
        curTaskSpec['type'] = slicerToGirderTypeMap[curType]
        curTaskSpec['format'] = slicerToGirderTypeMap[curType]

        if curType in ['image', 'file', 'directory']:
            curTaskSpec['target'] = 'filepath'  # check
            handlerDesc.param(curName + inputGirderSuffix,
                              'Girder ID of input %s - %s: %s'
                              % (curType, curName, curDesc),
                              dataType='string')
        else:
            handlerDesc.param(curName, curDesc, dataType='string')

        taskSpec['inputs'].append(curTaskSpec)

    # generate task spec for optional parameters
    for elt in paramXMLElements:
        curName = elt.findtext('name')
        curType = elt.tag
        curDesc = elt.findtext('description')

        curTaskSpec = dict()
        curTaskSpec['id'] = curName
        curTaskSpec['type'] = slicerToGirderTypeMap[curType]
        curTaskSpec['format'] = slicerToGirderTypeMap[curType]

        defaultValSpec = dict()
        defaultValSpec['format'] = curTaskSpec['format']
        strDefaultVal = elt.findtext('default')
        if strDefaultVal is not None:
            defaultVal = getDefaultValFromString(strDefaultVal,
                                                 curType)
        elif curType == 'boolean':
            defaultVal = False
        else:
            raise Exception(
                'optional parameters of type %s must '
                'provide a default value in the xml' % curType)
        defaultValSpec['data'] = defaultVal
        curTaskSpec['default'] = defaultValSpec

        handlerDesc.param(curName,
                          curDesc,
                          dataType='string',
                          required=False,
                          default=json.dumps(defaultVal))

        taskSpec['inputs'].append(curTaskSpec)

    # generate task spec for outputs
    for elt in outputXMLElements:
        curName = elt.findtext('name')
        curType = elt.tag
        curDesc = elt.findtext('description')

        # task spec for the current output
        curTaskSpec = dict()
        curTaskSpec['id'] = curName
        curTaskSpec['type'] = slicerToGirderTypeMap[curType]
        curTaskSpec['format'] = slicerToGirderTypeMap[curType]

        if curType in ['image', 'file', 'directory']:
            curTaskSpec['target'] = 'filepath'  # check

        taskSpec['outputs'].append(curTaskSpec)

        # param for parent folder
        handlerDesc.param(curName + outputGirderSuffix,
                          'Girder ID of parent folder '
                          'for output %s - %s: %s'
                          % (curType, curName, curDesc),
                          dataType='string')

        # param for name by which to store the current output
        handlerDesc.param(curName + outGirderNameSuffix,
                          'Name of output %s - %s: %s'
                          % (curType, curName, curDesc),
                          dataType='string')

    # define CLI handler function
    @boundHandler(restResource)
    @access.user
    @describeRoute(handlerDesc)
    def cliHandler(self, **args):

        user = self.getCurrentUser()
        token = self.getCurrentToken()['_id']

        # create job
        jobModel = self.model('job', 'jobs')
        jobTitle = '.'.join((restResource.resourceName, xmlName))
        job = jobModel.createJob(title=jobTitle,
                                 type=jobTitle,
                                 handler='worker_handler',
                                 user=user)
        kwargs = {
            'validate': False,
            'auto_convert': True,
            'cleanup': True}
        taskSpec['script'] = codeToRun

        # create job info
        jobToken = jobModel.createJobToken(job)
        kwargs['jobInfo'] = wutils.jobInfoSpec(job, jobToken)

        # create input bindings
        kwargs['inputs'] = dict()
        for elt in inputXMLElements:
            curName = elt.findtext('name')
            curType = elt.tag

            curBindingSpec = dict()
            if curType in ['image', 'file', 'directory']:
                # inputs of type image, file, or directory
                # should be located on girder
                if curType in ['image', 'file']:
                    curGirderType = 'item'
                else:
                    curGirderType = 'folder'
                curBindingSpec = wutils.girderInputSpec(
                    args[curName],
                    resourceType=curGirderType,
                    dataType='string', dataFormat='string',
                    token=token)
            else:
                # inputs that are not of type image, file, or directory
                # should be passed inline as string from json.dumps()
                curBindingSpec['mode'] = 'inline'
                curBindingSpec['type'] = slicerToGirderTypeMap[curType]
                curBindingSpec['format'] = 'json'
                curBindingSpec['data'] = args['params'][curName]

            kwargs['inputs'][curName] = curBindingSpec

        # create optional parameter bindings
        for elt in paramXMLElements:
            curName = elt.findtext('name')
            curType = elt.tag

            curBindingSpec = dict()
            curBindingSpec['mode'] = 'inline'
            curBindingSpec['type'] = slicerToGirderTypeMap[curType]
            curBindingSpec['format'] = 'json'
            curBindingSpec['data'] = args['params'][curName]

            kwargs['inputs'][curName] = curBindingSpec

        # create output boundings
        kwargs['outputs'] = dict()
        for elt in outputXMLElements:
            curName = elt.findtext('name')

            curBindingSpec = wutils.girderOutputSpec(
                args[curName],
                token,
                name=args['params'][curName + outGirderNameSuffix],
                dataType='string', dataFormat='string')
            kwargs['outputs'][curName] = curBindingSpec

            codeToSetOutputPath = "%s = os.path.join(_tempdir, '%s')" % (
                curName, args['params'][curName + outGirderNameSuffix])
            taskSpec['script'] = '\n'.join((codeToSetOutputPath,
                                           taskSpec['script']))

        # create task spec
        taskSpec['script'] = '\n'.join(("import os", taskSpec['script']))
        kwargs['task'] = taskSpec

        # schedule job
        job['kwargs'] = kwargs
        job = jobModel.save(job)
        jobModel.scheduleJob(job)

        # return result
        return jobModel.filter(job, user)

    handlerFunc = cliHandler

    # loadmodel stuff for inputs in girder
    for elt in inputXMLElements:
        curName = elt.findtext('name')
        curType = elt.tag

        if curType in ['image', 'file']:
            curModel = 'item'
        elif curType == 'directory':
            curModel = 'folder'
        else:
            continue
        curMap = {curName + inputGirderSuffix: curName}

        handlerFunc = loadmodel(map=curMap,
                                model=curModel,
                                level=AccessType.READ)(handlerFunc)

    # loadmodel stuff for outputs to girder
    for elt in outputXMLElements:
        curName = elt.findtext('name')

        curModel = 'folder'
        curMap = {curName + outputGirderSuffix: curName}

        handlerFunc = loadmodel(map=curMap,
                                model=curModel,
                                level=AccessType.WRITE)(handlerFunc)

    return handlerFunc

def genHandlerToGetCLIXmlSpec(restResource, xmlFile):
    """Generates a handler that returns the XML spec of the CLI

    Parameters
    ----------
    restResource : girder.api.rest.Resource
        The object of a class derived from girder.api.rest.Resource to which
        this handler will be attached
    xmlFile : str
        Full path to xml file of the CLI

    Returns
    -------
    function
        Returns a function that returns the xml spec of the CLI
    """

    xmlPath, xmlNameWExt = os.path.split(xmlFile)
    xmlName = os.path.splitext(xmlNameWExt)[0]

    # parse xml of cli
    clixml = etree.parse(xmlFile)

    # define the handler that returns the CLI's xml spec
    @boundHandler(restResource)
    @access.user
    @describeRoute(
        Description('Get XML spec of %s CLI' % xmlName)
    )
    def getXMLSpecHandler(self, *args, **kwargs):
        return etree.tostring(clixml)

    return getXMLSpecHandler

def genRESTEndPointsForSlicerCLIsInSubDirs(info, restResourceName, cliRootDir):
    """Generates REST end points for slicer CLIs placed in subdirectories of a
    given root directory and attaches them to a REST resource with the given
    name.

    For each CLI, it creates:
    * a GET Route (<apiURL>/`restResourceName`/<cliName>/xmlspec) that returns
    the xml spec of the CLI
    * a POST Route (<apiURL>/`restResourceName`/<cliName>/run) that runs
    the CLI

    It also creates a GET route (<apiURL>/`restResourceName`) that returns a
    list of relative routes to all CLIs attached to the generated REST resource

    Parameters
    ----------
    info
    restResourceName : str
        Name of the REST resource to which the end-points should be attached
    cliRootDir : str
        Full path of root directory containing the CLIs

    Returns
    -------

    """

    # create REST resource
    restResource = type(restResourceName,
                        (Resource, ),
                        {'resourceName': restResourceName})()

    # Add REST route for slicer CLIs located in subdirectories
    cliList = []

    for parentdir, dirnames, filenames in os.walk(cliRootDir):
        for subdir in dirnames:
            curCLIRelPath = os.path.relpath(os.path.join(parentdir, subdir),
                                            cliRootDir)

            # check if subdir contains a .xml file with the same name
            xmlFile = os.path.join(parentdir, subdir, subdir + '.xml')

            if not os.path.isfile(xmlFile):
                continue

            xmlPath, xmlNameWExt = os.path.split(xmlFile)
            xmlName = os.path.splitext(xmlNameWExt)[0]

            # check if subdir contains a .py file with the same name
            scriptFile = os.path.join(xmlPath, xmlName + '.py')

            if not os.path.isfile(scriptFile):
                continue

            print curCLIRelPath

            # TODO: check if xml adheres to slicer execution model xml schema

            # create a POST REST route that runs the CLI by invoking the handler
            try:
                cliRunHandler = genHandlerToRunCLI(restResource,
                                                   xmlFile, scriptFile)
            except Exception as e:
                print "Failed to create REST endpoints for %s: %s" % (
                    curCLIRelPath, e)
                continue

            cliRunHandlerName = 'run_' + xmlName
            setattr(restResource, cliRunHandlerName, cliRunHandler)
            restResource.route('POST',
                               (curCLIRelPath, 'run'),
                               getattr(restResource, cliRunHandlerName))

            # create GET REST route that returns the xml of the CLI
            try:
                cliGetXMLSpecHandler = genHandlerToGetCLIXmlSpec(restResource,
                                                                 xmlFile)
            except Exception as e:
                print "Failed to create REST endpoints for %s: %s" % (
                    curCLIRelPath, e)
                continue

            cliGetXMLSpecHandlerName = 'get_xml_' + xmlName
            setattr(restResource,
                    cliGetXMLSpecHandlerName,
                    cliGetXMLSpecHandler)
            restResource.route('GET',
                               (curCLIRelPath, 'xmlspec',),
                               getattr(restResource, cliGetXMLSpecHandlerName))

            cliList.append(curCLIRelPath)

    # create GET route that returns a list of relative routes to all CLIs
    @boundHandler(restResource)
    @access.user
    @describeRoute(
        Description('Get list of relative routes to all CLIs')
    )
    def getCLIListHandler(self, *args, **kwargs):
        return cliList

    getCLIListHandlerName = 'get_cli_list'
    setattr(restResource, getCLIListHandlerName, getCLIListHandler)
    restResource.route('GET', (), getattr(restResource, getCLIListHandlerName))

    # expose the generated REST resource via apiRoot
    setattr(info['apiRoot'], restResourceName, restResource)

def load(info):
    cliRootDir = os.path.dirname(__file__)
    genRESTEndPointsForSlicerCLIsInSubDirs(info, 'HistomicsTK', cliRootDir)
<|endoftext|>


<commit_before>import Map
tile_information = {
    'Farm': {
        'image': 'assets/hexagon-filled2.png',
        'build': {
            'energy': 1,
            'food': 1,
        },
        'consume': {
            'water': 1,
        },
        'produce': {
            'food': 3,
        }
    },
    'Habitat': {
        'image': 'assets/hexagon-filled2.png',
        'build': {
            'energy': 1,
            'food': 1,
        },
        'consume': {
            'food': 1,
        },
        'produce': {
            'energy': 2,
        }

    }
}

class GameData(object):

    def __init__(self, energy, water, food, people, map):
        self.resources = {
            'energy': energy,
            'water': water,
            'food': food
        }
        self.people = people
        self.map = map

    def build(self, type, x, y):
        hex = Map.Hex(x, y, tile_information[type])
        # check we have the right materials available to build
        if not self.have_enough_stuff(hex.build):
            return False
        # TODO: check the grid is alowed to be built on.
        # reduce amout of resources
        self.subtract_stuff(hex.build)

        # add tile to map
        self.map.add_hex(hex)
        return True

    def consume(self):
        result = {}
        # buildings
        for key, value in self.map.cost_of_hexes().iteritems():
            v = result.get(key, 0)
            result[key] = v + value

        # consue for people as well

        # check we have enough stuff to survive if not ???
        if not self.have_enough_stuff(result):
            return False
        # remove stuff from store.
        self.subtract_stuff(result)
        return True

    def produce(self):
        for key, value in self.map.gained_from_hexes().iteritems():
            self.resources[key] = self.resources[key] + value

    def have_enough_stuff(self, to_remove):
        for key, value in to_remove.iteritems():
            if self.resources[key] - value < 0:
                return False
        return True

    def subtract_stuff(self, to_remove):
        for key, value in to_remove.iteritems():
            self.resources[key] = self.resources[key] - value

    def print_resources(self):
        for key, value in self.resources.iteritems():
            print('{key}:{value}'.format(key=key, value=value))

    def addHabitatMaterial(self, n):
        self.habitatmaterial = self.habitatmaterial + int(n)

    def getHabitatMaterial(self):
        return self.habitatmaterial

    def getHabitats(self):
        return self.habitats

    def makeHabitats(self, number):
        n = int(number)
        self.habitatmaterial = self.habitatmaterial - n*self.habitatcost
        self.habitats = self.habitats + n
<commit_msg>Adding a build details method.<commit_after>import Map
tile_information = {
    'Farm': {
        'image': 'assets/hexagon-filled2.png',
        'build': {
            'energy': 1,
            'food': 1,
        },
        'consume': {
            'water': 1,
        },
        'produce': {
            'food': 3,
        }
    },
    'Habitat': {
        'image': 'assets/hexagon-filled2.png',
        'build': {
            'energy': 1,
            'food': 1,
        },
        'consume': {
            'food': 1,
        },
        'produce': {
            'energy': 2,
        }

    }
}

class GameData(object):

    def __init__(self, energy, water, food, people, map):
        self.resources = {
            'energy': energy,
            'water': water,
            'food': food
        }
        self.people = people
        self.map = map

    def build(self, type, x, y):
        hex = Map.Hex(x, y, tile_information[type])
        # check we have the right materials available to build
        if not self.have_enough_stuff(hex.build):
            return False
        # TODO: check the grid is alowed to be built on.
        # reduce amout of resources
        self.subtract_stuff(hex.build)

        # add tile to map
        self.map.add_hex(hex)
        return True

    def consume(self):
        result = {}
        # buildings
        for key, value in self.map.cost_of_hexes().iteritems():
            v = result.get(key, 0)
            result[key] = v + value

        # consue for people as well

        # check we have enough stuff to survive if not ???
        if not self.have_enough_stuff(result):
            return False
        # remove stuff from store.
        self.subtract_stuff(result)
        return True

    def buildCost(self, type)
        build_details = type['build']
        build_energy = build_details['energy']
        build_water = build_details['water']
        build_food = build_details['food']
        string = "Build cost is " + str(build_energy) + " energy and " + str(build_water) + " water and " + str(build_food) + "food"
        return string

    def produce(self):
        for key, value in self.map.gained_from_hexes().iteritems():
            self.resources[key] = self.resources[key] + value

    def have_enough_stuff(self, to_remove):
        for key, value in to_remove.iteritems():
            if self.resources[key] - value < 0:
                return False
        return True

    def subtract_stuff(self, to_remove):
        for key, value in to_remove.iteritems():
            self.resources[key] = self.resources[key] - value

    def print_resources(self):
        for key, value in self.resources.iteritems():
            print('{key}:{value}'.format(key=key, value=value))

    def addHabitatMaterial(self, n):
        self.habitatmaterial = self.habitatmaterial + int(n)

    def getHabitatMaterial(self):
        return self.habitatmaterial

    def getHabitats(self):
        return self.habitats

    def makeHabitats(self, number):
        n = int(number)
        self.habitatmaterial = self.habitatmaterial - n*self.habitatcost
        self.habitats = self.habitats + n
<|endoftext|>