<commit_before>import csv
import dataserv

def list_to_file(targ_list):
    with open('inv_list','a+') as tempfile:
        for i in range(0, len(targ_list)):
            tempfile.write(repr(targ_list[i]))
            tempfile.write(",")
            tempfile.write("\n")

def extend_list(targ_list):
    idinv_list = []
    for i in targ_list:
        idinv_list.extend(i['Id'])

def recurs_iter(target):
    if type(target) is list:
        src = recurs_iter(iter(target))

    return type(src)

def histogram():
    '''
    using bokeh to visualise:
    from bokeh.plotting import figure, output_file, show
    output_file('histogram.html')
    p = figure(title = 'insert title')
    x = datescount.keys()
    y = datescount.values()
    p.line(x,y)
    show(p)
    '''

    dates = eval(open('dates.txt', 'r+').read())
    from collections import Counter
    datescount = Counter(dates)

    return datescount

def sourcelist(cxn):
    testlist = [cxn.dates(), cxn.tags(), cxn.leadsources()]

    return testlist

def padlist(list1, list2):
    if len(list1)>len(list2):
        padding=(len(list1)-len(list2))*[0]
        list2.extend(padding)
    elif len(list2)>len(list1):
        padding=(len(list2-len(list1))*[0])
        list1.extend(padding)
    else:
        print("Didn't work, give me only 2 lists please")

    ziplist = zip(list1, list2)

    return ziplist

def better_datecompare(date1,date2):
    from datetime import datetime, timedelta
    date1=date1.timetuple()
    date2=date2.timetuple()
    date1=datetime(date1.tm_year,date1.tm_mon,date1.tm_mday)
    date2=datetime(date2.tm_year,date2.tm_mon,date2.tm_mday)
    days=date2-date1

    return days

def convert_infusiondate(IS_dateobject):
    from datetime import datetime
    date = IS_dateobject.timetuple()
    date = datetime(date.tm_year, date.tm_mon, date.tm_mday)

    return date

def linecount(filename):
    if type(filename) != str:
        filename = str(filename)

    with open(filename) as file:
        for i, l in enumerate(file):
            pass

        return i + 1

def leadtime_test():
    import dataserv as ds
    limit=dict(limit=1000)
    lts=ds.LeadtimetoSale().leadtime(**limit)
    ds.Process().procarray(lts)
    for item in lts:
        print(item)

def leadtime_test2():
    import dataserv as ds
    id_pages = ds.Query()._getpages('Contact','Id')
    limit = 999
    lts = []
    for page in range(0, id_pages + 1):
        ref_point = dict(page = page, limit = limit)
        lts.extend(ds.LeadtimetoSale().leadtime(**ref_point))
    ds.Process().procarray(lts)
    for item in lts:
        print(item)

def sendto_sqlite(query_array):
    '''Use sqlite3 module to output to local DB. Saves API calls. Using text datatypes
    in tables to avoid type conversion for datetime objects.
    '''
    import sqlite3
    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()
    c.execute('CREATE TABLE contacts (key text, value integer);')
    for item in query_array:
        # insert item into db. think about datatypes here
        # could possibly just write item as one whole string.
        # read it back in as dict later
        c.executemany('insert into contacts values (?,?);', item.iteritems())
    conn.commit()
    conn.close()

def sendto_json(query_array):
    '''Use json to store entire query as json file.'''
    import json
    with open('dataserv.json') as file:
        for item in diclist:
            json.dump(item, file)

def get_csv(filename):
    if type(filename) != str:
        filename = str(filename)
    csvdata = []
    with open(filename, newline = '') as csvfile:
        reader = csv.reader(csvfile, delimiter = ',')
        csvdata.extend([entry for entry in reader])
    return csvdata

def convert_invoice():
    '''Converts currency column in AUD to float.'''
    import locale
    import sqlite3
    locale.setlocale(locale.LC_ALL, '')
    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()
    c.execute('SELECT [Inv Total], rowid from sales;')
    invoices = c.fetchall()
    for row in invoices:
        invoices[invoices.index(row)] = list(row)
    for invoice in invoices:
        invoice[0] = invoice[0].strip('AUD')
        invoice[0] = invoice[0].strip('-AUD')
        invoice[0] = locale.atof(invoice[0])
    for row in invoices:
        invoices[invoices.index(row)] = tuple(row)
    c.executemany('UPDATE sales set [Inv Total]=? where rowid=?;', invoices)
    conn.commit()
    conn.close()

def create_joinlisttable():
    import sqlite3

    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()

    join_contacts_invoices = '''\
    SELECT contacts.Id, contacts.[Date Created], contacts.[Lead Source],\
    sales.[Inv Total], sales.Date \
    FROM contacts INNER JOIN sales \
    ON contacts.Id = sales.ContactId;\
    '''
    c.execute(join_contacts_invoices)
    joinlist = c.fetchall()
    joinlist.sort(key = lambda x: x[0])

    c.execute('''CREATE TABLE contactsales(
    contactid text, entrydate text, leadsource text, invamount text, invdate text);''')
    c.executemany('INSERT INTO contactsales VALUES (?,?,?,?,?);', joinlist)

    conn.commit()
    conn.close()

def get_invoicedates():
    import sqlite3

    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()
    conn.text_factory = int
    c.execute('SELECT Id FROM contacts;')
    contact_idlist = c.fetchall()
    contact_invlist = dict()
    conn.text_factory = str
    for cid in contact_idlist:
        c.execute('SELECT Date FROM sales where sales.ContactId = (?);', cid)
        contact_invlist[cid] = c.fetchall()
    conn.close()
    return contact_invlist

def leadtime_fromdb(datecreated, invdates):
    '''
    datecreated: contact creation date
    invdates: list of invoice dates for same contact
    '''

    dt_creat = convert_datestring(datecreated)
    new_invdates = [convert_datestring(date) for date in invdates]
    firstsale = min(new_invdates)
    leadtime = firstsale - dt_creat

    return leadtime  # this will be number of days

def convert_datestring(targetdate):
    import time
    newdate = time.strptime(targetdate.split()[0], '%d/%m/%Y')
    newdate = time.mktime(newdate)

    return newdate

def list_convert(targetlist):
    newlist = [list(row) for row in targetlist]
    for newrow in newlist:
        newrow[1] = convert_datestring(newrow[1])
        newrow[4] = convert_datestring(newrow[4])

    return newlist

def get_db_table(db_name, db_table):
    import sqlite3

    conn = sqlite3.connect(db_name)
    c = conn.cursor()
    c.execute('SELECT * FROM {}'.format(db_table))
    db_tbl = c.fetchall()

    return db_tbl

def leadtime_from_db(targetlist):
    newlist = dict()
    for row in targetlist:
        if row[0] not in newlist.keys():
            newlist[row[0]] = dict(entrydate = row[1], invdates = [row[4]])
        else:
            newlist[row[0]]['invdate'].append(row[4])

        leadtime = min(newlist['invdate']) - newlist['entrydate']
        newlist[row[0]]['leadtime'] = leadtime

    return newlist
<commit_msg>Wrap up leadtime processing in get_leadtime() function. Returns dict of dicts. Yeah baby.<commit_after>import csv
import dataserv

def list_to_file(targ_list):
    with open('inv_list','a+') as tempfile:
        for i in range(0, len(targ_list)):
            tempfile.write(repr(targ_list[i]))
            tempfile.write(",")
            tempfile.write("\n")

def extend_list(targ_list):
    idinv_list = []
    for i in targ_list:
        idinv_list.extend(i['Id'])

def recurs_iter(target):
    if type(target) is list:
        src = recurs_iter(iter(target))

    return type(src)

def histogram():
    '''
    using bokeh to visualise:
    from bokeh.plotting import figure, output_file, show
    output_file('histogram.html')
    p = figure(title = 'insert title')
    x = datescount.keys()
    y = datescount.values()
    p.line(x,y)
    show(p)
    '''

    dates = eval(open('dates.txt', 'r+').read())
    from collections import Counter
    datescount = Counter(dates)

    return datescount

def sourcelist(cxn):
    testlist = [cxn.dates(), cxn.tags(), cxn.leadsources()]

    return testlist

def padlist(list1, list2):
    if len(list1)>len(list2):
        padding=(len(list1)-len(list2))*[0]
        list2.extend(padding)
    elif len(list2)>len(list1):
        padding=(len(list2-len(list1))*[0])
        list1.extend(padding)
    else:
        print("Didn't work, give me only 2 lists please")

    ziplist = zip(list1, list2)

    return ziplist

def better_datecompare(date1,date2):
    from datetime import datetime, timedelta
    date1=date1.timetuple()
    date2=date2.timetuple()
    date1=datetime(date1.tm_year,date1.tm_mon,date1.tm_mday)
    date2=datetime(date2.tm_year,date2.tm_mon,date2.tm_mday)
    days=date2-date1

    return days

def convert_infusiondate(IS_dateobject):
    from datetime import datetime
    date = IS_dateobject.timetuple()
    date = datetime(date.tm_year, date.tm_mon, date.tm_mday)

    return date

def linecount(filename):
    if type(filename) != str:
        filename = str(filename)

    with open(filename) as file:
        for i, l in enumerate(file):
            pass

        return i + 1

def leadtime_test():
    import dataserv as ds
    limit=dict(limit=1000)
    lts=ds.LeadtimetoSale().leadtime(**limit)
    ds.Process().procarray(lts)
    for item in lts:
        print(item)

def leadtime_test2():
    import dataserv as ds
    id_pages = ds.Query()._getpages('Contact','Id')
    limit = 999
    lts = []
    for page in range(0, id_pages + 1):
        ref_point = dict(page = page, limit = limit)
        lts.extend(ds.LeadtimetoSale().leadtime(**ref_point))
    ds.Process().procarray(lts)
    for item in lts:
        print(item)

def sendto_sqlite(query_array):
    '''Use sqlite3 module to output to local DB. Saves API calls. Using text datatypes
    in tables to avoid type conversion for datetime objects.
    '''
    import sqlite3
    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()
    c.execute('CREATE TABLE contacts (key text, value integer);')
    for item in query_array:
        # insert item into db. think about datatypes here
        # could possibly just write item as one whole string.
        # read it back in as dict later
        c.executemany('insert into contacts values (?,?);', item.iteritems())
    conn.commit()
    conn.close()

def sendto_json(query_array):
    '''Use json to store entire query as json file.'''
    import json
    with open('dataserv.json') as file:
        for item in diclist:
            json.dump(item, file)

def get_csv(filename):
    if type(filename) != str:
        filename = str(filename)
    csvdata = []
    with open(filename, newline = '') as csvfile:
        reader = csv.reader(csvfile, delimiter = ',')
        csvdata.extend([entry for entry in reader])
    return csvdata

def convert_invoice():
    '''Converts currency column in AUD to float.'''
    import locale
    import sqlite3
    locale.setlocale(locale.LC_ALL, '')
    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()
    c.execute('SELECT [Inv Total], rowid from sales;')
    invoices = c.fetchall()
    for row in invoices:
        invoices[invoices.index(row)] = list(row)
    for invoice in invoices:
        invoice[0] = invoice[0].strip('AUD')
        invoice[0] = invoice[0].strip('-AUD')
        invoice[0] = locale.atof(invoice[0])
    for row in invoices:
        invoices[invoices.index(row)] = tuple(row)
    c.executemany('UPDATE sales set [Inv Total]=? where rowid=?;', invoices)
    conn.commit()
    conn.close()

def create_joinlisttable():
    import sqlite3

    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()

    join_contacts_invoices = '''\
    SELECT contacts.Id, contacts.[Date Created], contacts.[Lead Source],\
    sales.[Inv Total], sales.Date \
    FROM contacts INNER JOIN sales \
    ON contacts.Id = sales.ContactId;\
    '''
    c.execute(join_contacts_invoices)
    joinlist = c.fetchall()
    joinlist.sort(key = lambda x: x[0])

    c.execute('''CREATE TABLE contactsales(
    contactid text, entrydate text, leadsource text, invamount text, invdate text);''')
    c.executemany('INSERT INTO contactsales VALUES (?,?,?,?,?);', joinlist)

    conn.commit()
    conn.close()

def get_invoicedates():
    import sqlite3

    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()
    conn.text_factory = int
    c.execute('SELECT Id FROM contacts;')
    contact_idlist = c.fetchall()
    contact_invlist = dict()
    conn.text_factory = str
    for cid in contact_idlist:
        c.execute('SELECT Date FROM sales where sales.ContactId = (?);', cid)
        contact_invlist[cid] = c.fetchall()
    conn.close()
    return contact_invlist

def leadtime_fromdb(datecreated, invdates):
    '''
    datecreated: contact creation date
    invdates: list of invoice dates for same contact
    '''

    dt_creat = convert_datestring(datecreated)
    new_invdates = [convert_datestring(date) for date in invdates]
    firstsale = min(new_invdates)
    leadtime = firstsale - dt_creat

    return leadtime  # this will be number of days

def convert_datestring(targetdate):
    import time
    newdate = time.strptime(targetdate.split()[0], '%d/%m/%Y')
    newdate = time.mktime(newdate)

    return newdate

def list_convert(targetlist):
    newlist = [list(row) for row in targetlist]
    for newrow in newlist:
        newrow[1] = convert_datestring(newrow[1])
        newrow[4] = convert_datestring(newrow[4])

    return newlist

def get_db_table(db_name, db_table):
    import sqlite3

    conn = sqlite3.connect(db_name)
    c = conn.cursor()
    c.execute('SELECT * FROM {}'.format(db_table))
    db_tbl = c.fetchall()

    return db_tbl

def leadtime_from_db(targetlist):
    newlist = dict()
    for row in targetlist:
        if row[0] not in newlist.keys():
            newlist[row[0]] = dict(entrydate = row[1], invdates = [row[4]])
        else:
            newlist[row[0]]['invdates'].append(row[4])

        leadtime = min(newlist[row[0]]['invdates']) - newlist[row[0]]['entrydate']
        newlist[row[0]]['leadtime'] = leadtime

    return newlist

def get_leadtime():
    data = get_db_table('dataserv.db', 'contactsales')
    data = list_convert(data)
    data = leadtime_from_db(data)

    return data
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# For debugging
# NVIM_PYTHON_LOG_FILE=nvim.log NVIM_PYTHON_LOG_LEVEL=INFO nvim

# detach 1 for quick shutdown for neovim, detach 0, 'cause jedi enters infinite
# loops sometime, don't know why.
from cm import get_src, register_source
register_source(name='cm-jedi',
                   priority=9,
                   abbreviation='Py',
                   scoping=True,
                   scopes=['python'],
                   events=['InsertLeave'],
                   detach=0)

import os
import re
import logging
import jedi
from neovim import attach, setup_logging

logger = logging.getLogger(__name__)

class Source:

    def __init__(self,nvim):

        self._nvim = nvim

    def cm_refresh(self,info,ctx,*args):

        lnum = ctx['lnum']
        col = ctx['col']
        typed = ctx['typed']
        path = ctx['filepath']

        kwtyped = re.search(r'[0-9a-zA-Z_]*?$',typed).group(0)
        startcol = col-len(kwtyped)

        src = get_src(self._nvim,ctx)
        if not src.strip():
            # empty src may possibly block jedi execution, don't know why
            logger.info('ignore empty src [%s]', src)
            return

        skip_completions = False
        show_sig = False

        # completion pattern
        if (re.search(r'^(import|from)', typed)
            or re.search(r'[\w_]{2,}$',typed)
            or re.search(r'\.[\w_]*$',typed)
            ):
            skip_completions = False
        else:
            skip_completions = True
            if kwtyped=="":
                show_sig = True
            else:
                # skip and no show sig, no need to process
                return

        # logger.info('jedi.Script lnum[%s] curcol[%s] path[%s] [%s]', lnum,len(typed),path,src)
        script = jedi.Script(src, lnum, len(typed), path)

        if show_sig:
            signature_text = self._get_signature_text(script)
            if signature_text:
                matches = [dict(word='',empty=1,abbr=signature_text,dup=1),]
                self._nvim.call('cm#complete', info['name'], ctx, col, matches, True, async=True)
            return

        if skip_completions:
            return

        completions = script.completions()
        logger.info('completions %s', completions)

        matches = []

        for complete in completions:

            item = dict(word=kwtyped+complete.complete,
                        icase=1,
                        dup=1,
                        menu=complete.description,
                        info=complete.docstring()
                        )
            # Fix the user typed case
            if item['word'].lower()==complete.name.lower():
                item['word'] = complete.name
            matches.append(item)

        # cm#complete(src, context, startcol, matches)
        ret = self._nvim.call('cm#complete', info['name'], ctx, startcol, matches, async=True)
        logger.info('matches %s, ret %s', matches, ret)

    def _get_signature_text(self,script):
        signature_text = ''
        # TODO: optimize
        # currently simply use the last signature
        signatures = script.call_signatures()
        logger.info('signatures: %s', signatures)
        if len(signatures)>0:
            signature = signatures[-1]
            params=[param.description for param in signature.params]
            signature_text = signature.name + '(' + ', '.join(params) + ')'
        return signature_text

<commit_msg>configurable jedi source pattern<commit_after># -*- coding: utf-8 -*-

# For debugging
# NVIM_PYTHON_LOG_FILE=nvim.log NVIM_PYTHON_LOG_LEVEL=INFO nvim

# detach 1 for quick shutdown for neovim, detach 0, 'cause jedi enters infinite
# loops sometime, don't know why.
from cm import get_src, register_source
register_source(name='cm-jedi',
                   priority=9,
                   abbreviation='Py',
                   scoping=True,
                   scopes=['python'],
                   cm_refresh_patterns=[r'^(import|from) ', r'[\w_]{3,}$',r'\.[\w_]*$',r'\(|\,\s*$'],
                   detach=0)

import os
import re
import logging
import jedi
from neovim import attach, setup_logging

logger = logging.getLogger(__name__)

class Source:

    def __init__(self,nvim):

        self._nvim = nvim

    def cm_refresh(self,info,ctx,*args):

        lnum = ctx['lnum']
        col = ctx['col']
        typed = ctx['typed']
        path = ctx['filepath']

        kwtyped = re.search(r'[0-9a-zA-Z_]*?$',typed).group(0)
        startcol = col-len(kwtyped)

        src = get_src(self._nvim,ctx)
        if not src.strip():
            # empty src may possibly block jedi execution, don't know why
            logger.info('ignore empty src [%s]', src)
            return

        show_sig = False
        # show function signature when `(` or `,` is typed
        if re.search(r'\(|\,\s*$', typed):
            show_sig = True

        # logger.info('jedi.Script lnum[%s] curcol[%s] path[%s] [%s]', lnum,len(typed),path,src)
        script = jedi.Script(src, lnum, len(typed), path)

        if show_sig:
            signature_text = self._get_signature_text(script)
            if signature_text:
                matches = [dict(word='',empty=1,abbr=signature_text,dup=1),]
                self._nvim.call('cm#complete', info['name'], ctx, col, matches, True, async=True)
            return

        completions = script.completions()
        logger.info('completions %s', completions)

        matches = []

        for complete in completions:

            item = dict(word=kwtyped+complete.complete,
                        icase=1,
                        dup=1,
                        menu=complete.description,
                        info=complete.docstring()
                        )
            # Fix the user typed case
            if item['word'].lower()==complete.name.lower():
                item['word'] = complete.name
            matches.append(item)

        # cm#complete(src, context, startcol, matches)
        ret = self._nvim.call('cm#complete', info['name'], ctx, startcol, matches, async=True)
        logger.info('matches %s, ret %s', matches, ret)

    def _get_signature_text(self,script):
        signature_text = ''
        # TODO: optimize
        # currently simply use the last signature
        signatures = script.call_signatures()
        logger.info('signatures: %s', signatures)
        if len(signatures)>0:
            signature = signatures[-1]
            params=[param.description for param in signature.params]
            signature_text = signature.name + '(' + ', '.join(params) + ')'
        return signature_text

<|endoftext|>


<commit_before>import os
import time
import logging
import importlib
from cached_property import cached_property
from gc3libs import Run
from gc3libs.workflow import SequentialTaskCollection
from gc3libs.workflow import StopOnError
from ..errors import WorkflowNextStepError

logger = logging.getLogger(__name__)

def load_program(prog_name):
    '''
    Load a `TissueMAPS` command line program.

    Parameters
    ----------
    prog_name: str
        name of the program, i.e. the name of corresponding subpackage in the
        "tmlib" package

    Returns
    -------
    tmlib.cli.CommandLineInterface
        command line program

    Raises
    ------
    ImportError
        when subpackage with name `prog_name` doesn't have a module named "cli"
    AttributeError
        when the "cli" module doesn't contain a program-specific
        implementation of the `CommandLineInterface` base class
    '''
    module_name = 'tmlib.%s.cli' % prog_name
    logger.debug('load cli module "%s"' % module_name)
    module = importlib.import_module(module_name)
    class_name = prog_name.capitalize()
    return getattr(module, class_name)

def load_method_args(method_name):
    '''
    Load general arguments that can be parsed to a method of
    an implemented subclass of a :py:class:`tmlib.cli.CommandLineInterface`
    base class

    Parameters
    ----------
    method_name: str
        name of the method

    Returns
    -------
    tmlib.args.Args
        argument container

    Raises
    ------
    AttrbuteError
        when the "args" module doesn't contain a method-specific
        implementation of the `Args` base class
    '''
    module_name = 'tmlib.args'
    module = importlib.import_module(module_name)
    class_name = '%sArgs' % method_name.capitalize()
    return getattr(module, class_name)

def load_var_method_args(prog_name, method_name):
    '''
    Load variable program-specific arguments that can be parsed to
    a method of an implemented subclass of a
    :py:class:`tmlib.cli.CommandLineInterface` base class.

    Parameters
    ----------
    prog_name: str
        name of the program
    method_name: str
        name of the method

    Returns
    -------
    tmlib.args.Args
        argument container

    Note
    ----
    Returns ``None`` when the "args" module in the subpackage with name
    `prog_name` doesn't contain a program- and method-specific implementation
    of the `Args` base class.

    Raises
    ------
    ImportError
        when subpackage with name `prog_name` doesn't have a module named "args"
    '''
    package_name = 'tmlib.%s' % prog_name
    module_name = 'tmlib.%s.args' % prog_name
    importlib.import_module(package_name)
    module = importlib.import_module(module_name)
    class_name = '%s%sArgs' % (prog_name.capitalize(),
                               method_name.capitalize())
    try:
        return getattr(module, class_name)
    except AttributeError:
        return None

class Workflow(SequentialTaskCollection, StopOnError):

    def __init__(self, experiment, verbosity, description=None,
                 start_stage=None, start_step=None):
        '''
        Initialize an instance of class Workflow.

        Parameters
        ----------
        experiment: str
            configured experiment object
        verbosity: int
            logging verbosity level
        description: tmlib.cfg.WorkflowDescription, optional
            description of the workflow that should be (default: ``None``)
        start_stage: str or int, optional
            name or index of a stage from where the workflow should be
            started (default: ``None``)
        start_step: str or int, optional
            name or index of a step within `stage` from where the workflow
            should be started (default: ``None``)

        Note
        ----
        If no `description` is provided, the description is
        obtained from the user configuration settings file.

        See also
        --------
        :py:mod:`tmlib.cfg`
        '''
        super(Workflow, self).__init__(tasks=None, jobname='tmaps')
        self.experiment = experiment
        self.workflow = description
        if self.workflow is None:
            self.workflow = self.experiment.user_cfg.workflow
        self.start_stage = start_stage
        self.start_step = start_step
        self.tasks = list()
        self.expected_outputs = list()
        self.verbosity = verbosity
        self._add_step(0)

    @cached_property
    def steps_to_process(self):
        '''
        Returns
        -------
        List[WorkflowStepDescription]
            description of each step of the workflow that should be processed

        Note
        ----
        Arguments can be set in the user configuration file.
        '''
        steps_to_process = list()
        if self.start_stage is None:
            stage_ix = 0
        elif isinstance(self.start_stage, basestring):
            stage_names = [s.name for s in self.workflow.stages]
            stage_ix = stage_names.index(self.start_stage)
        elif isinstance(self.start_stage, int):
            stage_ix = self.start_stage
        else:
            raise TypeError('Argument "start_stage" must have type str or int.')
        stage = self.workflow.stages[stage_ix]
        if self.start_step is None:
            step_ix = 0
        elif isinstance(self.start_step, basestring):
            step_names = [s.name for s in stage.steps]
            step_ix = step_names.index(self.start_step)
        elif isinstance(self.start_step, int):
            step_ix = self.start_step
        else:
            raise TypeError('Argument "start_step" must have type str or int.')
        logger.info('start workflow at stage "%s" step "%s"',
                    stage.name, stage.steps[step_ix].name)
        for i, stage in enumerate(self.workflow.stages):
            if i < stage_ix:
                continue
            for j, step in enumerate(stage.steps):
                if i == stage_ix and j < step_ix:
                    continue
                logger.debug('add step "%s" to workflow', step.name)
                steps_to_process.append(step)
        return steps_to_process

    def _create_jobs_for_next_step(self, step):
        logger.debug('create jobs for step {0}'.format(step.name))
        prog_name = step.name
        logger.debug('load program "%s"', prog_name)
        prog = load_program(prog_name)
        logger.debug('create a program instance')
        prog_instance = prog(self.experiment, self.workflow.verbosity)

        logger.debug('call "init" method with configured arguments')
        prog_instance.init(step.args)
        # Check whether inputs of current step were generated by previous steps
        if not all([
                    os.path.exists(i)
                    for i in prog_instance.required_inputs
                ]):
            logger.error('required inputs were not generated')
            raise WorkflowNextStepError('required inputs do not exist')

        # Store the expected outputs to be later able to check whether they
        # were actually generated
        self.expected_outputs.append(prog_instance.expected_outputs)

        logger.debug('build GC3Pie jobs')
        jobs = prog_instance.build_jobs(virtualenv=self.workflow.virtualenv)
        # jobs: gc3libs.workflow.SequentialTaskCollection
        return jobs

    def _add_step(self, index):
        if index > 0:
            if not all([os.path.exists(f) for f in self.expected_outputs[-1]]):
                logger.error('expected outputs were not generated')
                raise WorkflowNextStepError(
                             'outputs of previous step do not exist')
        logger.debug('create job descriptions for next step')
        task = self._create_jobs_for_next_step(self.steps_to_process[index])
        logger.debug('add jobs to the workflow task list')
        self.tasks.append(task)

    def next(self, done):
        '''
        Progress to the next step of the workflow.

        Parameters
        ----------
        done: int
            zero-based index of the last processed step

        Returns
        -------
        gc3libs.Run.State
        '''
        if done+1 < len(self.steps_to_process):
            logger.info('waiting to give NFS time to get up-to-date')
            time.sleep(60)
            try:
                logger.info('progress to next step ({0} of {1}): "{2}"'.format(
                                (done+1)+1, len(self.steps_to_process),
                                self.steps_to_process[done+1].name))
                self._add_step(done+1)
            except Exception as error:
                logger.error('adding next step failed: %s', error)
                self.execution.exitcode = 1
                logger.debug('set exitcode to one')
                logger.debug('set state to TERMINATED')
                return Run.State.TERMINATED

        return super(Workflow, self).next(done)
<commit_msg>Fix bug in tmaps workflow class<commit_after>import os
import time
import logging
import importlib
from cached_property import cached_property
from gc3libs import Run
from gc3libs.workflow import SequentialTaskCollection
from gc3libs.workflow import StopOnError
from ..errors import WorkflowNextStepError

logger = logging.getLogger(__name__)

def load_program(prog_name):
    '''
    Load a `TissueMAPS` command line program.

    Parameters
    ----------
    prog_name: str
        name of the program, i.e. the name of corresponding subpackage in the
        "tmlib" package

    Returns
    -------
    tmlib.cli.CommandLineInterface
        command line program

    Raises
    ------
    ImportError
        when subpackage with name `prog_name` doesn't have a module named "cli"
    AttributeError
        when the "cli" module doesn't contain a program-specific
        implementation of the `CommandLineInterface` base class
    '''
    module_name = 'tmlib.%s.cli' % prog_name
    logger.debug('load cli module "%s"' % module_name)
    module = importlib.import_module(module_name)
    class_name = prog_name.capitalize()
    return getattr(module, class_name)

def load_method_args(method_name):
    '''
    Load general arguments that can be parsed to a method of
    an implemented subclass of a :py:class:`tmlib.cli.CommandLineInterface`
    base class

    Parameters
    ----------
    method_name: str
        name of the method

    Returns
    -------
    tmlib.args.Args
        argument container

    Raises
    ------
    AttrbuteError
        when the "args" module doesn't contain a method-specific
        implementation of the `Args` base class
    '''
    module_name = 'tmlib.args'
    module = importlib.import_module(module_name)
    class_name = '%sArgs' % method_name.capitalize()
    return getattr(module, class_name)

def load_var_method_args(prog_name, method_name):
    '''
    Load variable program-specific arguments that can be parsed to
    a method of an implemented subclass of a
    :py:class:`tmlib.cli.CommandLineInterface` base class.

    Parameters
    ----------
    prog_name: str
        name of the program
    method_name: str
        name of the method

    Returns
    -------
    tmlib.args.Args
        argument container

    Note
    ----
    Returns ``None`` when the "args" module in the subpackage with name
    `prog_name` doesn't contain a program- and method-specific implementation
    of the `Args` base class.

    Raises
    ------
    ImportError
        when subpackage with name `prog_name` doesn't have a module named "args"
    '''
    package_name = 'tmlib.%s' % prog_name
    module_name = 'tmlib.%s.args' % prog_name
    importlib.import_module(package_name)
    module = importlib.import_module(module_name)
    class_name = '%s%sArgs' % (prog_name.capitalize(),
                               method_name.capitalize())
    try:
        return getattr(module, class_name)
    except AttributeError:
        return None

class Workflow(SequentialTaskCollection, StopOnError):

    def __init__(self, experiment, verbosity, description=None,
                 start_stage=None, start_step=None):
        '''
        Initialize an instance of class Workflow.

        Parameters
        ----------
        experiment: str
            configured experiment object
        verbosity: int
            logging verbosity level
        description: tmlib.cfg.WorkflowDescription, optional
            description of the workflow that should be (default: ``None``)
        start_stage: str or int, optional
            name or index of a stage from where the workflow should be
            started (default: ``None``)
        start_step: str or int, optional
            name or index of a step within `stage` from where the workflow
            should be started (default: ``None``)

        Note
        ----
        If no `description` is provided, the description is
        obtained from the user configuration settings file.

        See also
        --------
        :py:mod:`tmlib.cfg`
        '''
        super(Workflow, self).__init__(tasks=None, jobname='tmaps')
        self.experiment = experiment
        self.workflow = description
        if self.workflow is None:
            self.workflow = self.experiment.user_cfg.workflow
        self.start_stage = start_stage
        self.start_step = start_step
        self.tasks = list()
        self.expected_outputs = list()
        self.verbosity = verbosity
        self._add_step(0)

    @cached_property
    def steps_to_process(self):
        '''
        Returns
        -------
        List[WorkflowStepDescription]
            description of each step of the workflow that should be processed

        Note
        ----
        Arguments can be set in the user configuration file.
        '''
        steps_to_process = list()
        if self.start_stage is None:
            stage_ix = 0
        elif isinstance(self.start_stage, basestring):
            stage_names = [s.name for s in self.workflow.stages]
            stage_ix = stage_names.index(self.start_stage)
        elif isinstance(self.start_stage, int):
            stage_ix = self.start_stage
        else:
            raise TypeError('Argument "start_stage" must have type str or int.')
        stage = self.workflow.stages[stage_ix]
        if self.start_step is None:
            step_ix = 0
        elif isinstance(self.start_step, basestring):
            step_names = [s.name for s in stage.steps]
            step_ix = step_names.index(self.start_step)
        elif isinstance(self.start_step, int):
            step_ix = self.start_step
        else:
            raise TypeError('Argument "start_step" must have type str or int.')
        logger.info('start workflow at stage "%s" step "%s"',
                    stage.name, stage.steps[step_ix].name)
        for i, stage in enumerate(self.workflow.stages):
            if i < stage_ix:
                continue
            for j, step in enumerate(stage.steps):
                if i == stage_ix and j < step_ix:
                    continue
                logger.debug('add step "%s" to workflow', step.name)
                steps_to_process.append(step)
        return steps_to_process

    def _create_jobs_for_next_step(self, step):
        logger.debug('create jobs for step {0}'.format(step.name))
        prog_name = step.name
        logger.debug('load program "%s"', prog_name)
        prog = load_program(prog_name)
        logger.debug('create a program instance')
        prog_instance = prog(self.experiment, self.verbosity)

        logger.debug('call "init" method with configured arguments')
        prog_instance.init(step.args)
        # Check whether inputs of current step were generated by previous steps
        if not all([
                    os.path.exists(i)
                    for i in prog_instance.required_inputs
                ]):
            logger.error('required inputs were not generated')
            raise WorkflowNextStepError('required inputs do not exist')

        # Store the expected outputs to be later able to check whether they
        # were actually generated
        self.expected_outputs.append(prog_instance.expected_outputs)

        logger.debug('build GC3Pie jobs')
        jobs = prog_instance.build_jobs(virtualenv=self.workflow.virtualenv)
        # jobs: gc3libs.workflow.SequentialTaskCollection
        return jobs

    def _add_step(self, index):
        if index > 0:
            if not all([os.path.exists(f) for f in self.expected_outputs[-1]]):
                logger.error('expected outputs were not generated')
                raise WorkflowNextStepError(
                             'outputs of previous step do not exist')
        logger.debug('create job descriptions for next step')
        task = self._create_jobs_for_next_step(self.steps_to_process[index])
        logger.debug('add jobs to the workflow task list')
        self.tasks.append(task)

    def next(self, done):
        '''
        Progress to the next step of the workflow.

        Parameters
        ----------
        done: int
            zero-based index of the last processed step

        Returns
        -------
        gc3libs.Run.State
        '''
        if done+1 < len(self.steps_to_process):
            logger.info('waiting to give NFS time to get up-to-date')
            time.sleep(60)
            try:
                logger.info('progress to next step ({0} of {1}): "{2}"'.format(
                                (done+1)+1, len(self.steps_to_process),
                                self.steps_to_process[done+1].name))
                self._add_step(done+1)
            except Exception as error:
                logger.error('adding next step failed: %s', error)
                self.execution.exitcode = 1
                logger.debug('set exitcode to one')
                logger.debug('set state to TERMINATED')
                return Run.State.TERMINATED

        return super(Workflow, self).next(done)
<|endoftext|>


<commit_before>"""Package info"""

__version__ = '0.5.3.1'
__author__ = ' William Falcon et al.'
__author_email__ = 'waf2107@columbia.edu'
__license__ = 'Apache-2.0'
__homepage__ = 'https://github.com/williamFalcon/pytorch-lightning'
# this has to be simple string, see: https://github.com/pypa/twine/issues/522
__docs__ = "PyTorch Lightning is the lightweight PyTorch wrapper for ML researchers." \
           " Scale your models. Write less boilerplate."

try:
    # This variable is injected in the __builtins__ by the build
    # process. It used to enable importing subpackages of skimage when
    # the binaries are not built
    __LIGHTNING_SETUP__
except NameError:
    __LIGHTNING_SETUP__ = False

if __LIGHTNING_SETUP__:
    import sys
    sys.stderr.write('Partial import of skimage during the build process.\n')
    # We are not importing the rest of the scikit during the build
    # process, as it may not be compiled yet
else:
    from .trainer.trainer import Trainer
    from .root_module.root_module import LightningModule
    from .root_module.decorators import data_loader

    __all__ = [
        'Trainer',
        'LightningModule',
        'data_loader',
    ]
<commit_msg>release v0.5.3.2<commit_after>"""Package info"""

__version__ = '0.5.3.2'
__author__ = ' William Falcon et al.'
__author_email__ = 'waf2107@columbia.edu'
__license__ = 'Apache-2.0'
__homepage__ = 'https://github.com/williamFalcon/pytorch-lightning'
# this has to be simple string, see: https://github.com/pypa/twine/issues/522
__docs__ = "PyTorch Lightning is the lightweight PyTorch wrapper for ML researchers." \
           " Scale your models. Write less boilerplate."

try:
    # This variable is injected in the __builtins__ by the build
    # process. It used to enable importing subpackages of skimage when
    # the binaries are not built
    __LIGHTNING_SETUP__
except NameError:
    __LIGHTNING_SETUP__ = False

if __LIGHTNING_SETUP__:
    import sys
    sys.stderr.write('Partial import of skimage during the build process.\n')
    # We are not importing the rest of the scikit during the build
    # process, as it may not be compiled yet
else:
    from .trainer.trainer import Trainer
    from .root_module.root_module import LightningModule
    from .root_module.decorators import data_loader

    __all__ = [
        'Trainer',
        'LightningModule',
        'data_loader',
    ]
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import django
from future.builtins import int, zip

from functools import reduce
from operator import ior, iand
from string import punctuation

from django.apps import apps
from django.core.exceptions import ImproperlyConfigured
from django.db.models import Manager, Q, CharField, TextField
from django.db.models.manager import ManagerDescriptor
from django.db.models.query import QuerySet
from django.contrib.sites.managers import CurrentSiteManager as DjangoCSM
from django.utils.timezone import now
from django.utils.translation import ugettext_lazy as _

from mezzanine.conf import settings
from mezzanine.utils.sites import current_site_id
from mezzanine.utils.urls import home_slug

if django.VERSION >= (1, 10):
    class ManagerDescriptor(ManagerDescriptor):
        """
        This class exists purely to skip the abstract model check
        in the __get__ method of Django's ManagerDescriptor.
        """
        def __get__(self, instance, cls=None):
            if instance is not None:
                raise AttributeError(
                    "Manager isn't accessible via %s instances" % cls.__name__
                )

            # In ManagerDescriptor.__get__, an exception is raised here
            # if cls is abstract

            if cls._meta.swapped:
                raise AttributeError(
                    "Manager isn't available; "
                    "'%s.%s' has been swapped for '%s'" % (
                        cls._meta.app_label,
                        cls._meta.object_name,
                        cls._meta.swapped,
                    )
                )

            return cls._meta.managers_map[self.manager.name]

class PublishedManager(Manager):
    """
    Provides filter for restricting items returned by status and
    publish date when the given user is not a staff member.
    """

    def published(self, for_user=None):
        """
        For non-staff users, return items with a published status and
        whose publish and expiry dates fall before and after the
        current date when specified.
        """
        from mezzanine.core.models import CONTENT_STATUS_PUBLISHED
        if for_user is not None and for_user.is_staff:
            return self.all()
        return self.filter(
            Q(publish_date__lte=now()) | Q(publish_date__isnull=True),
            Q(expiry_date__gte=now()) | Q(expiry_date__isnull=True),
            Q(status=CONTENT_STATUS_PUBLISHED))

    def get_by_natural_key(self, slug):
        return self.get(slug=slug)

def search_fields_to_dict(fields):
    """
    In ``SearchableQuerySet`` and ``SearchableManager``, search fields
    can either be a sequence, or a dict of fields mapped to weights.
    This function converts sequences to a dict mapped to even weights,
    so that we're consistently dealing with a dict of fields mapped to
    weights, eg: ("title", "content") -> {"title": 1, "content": 1}
    """
    if not fields:
        return {}
    try:
        int(list(dict(fields).values())[0])
    except (TypeError, ValueError):
        fields = dict(zip(fields, [1] * len(fields)))
    return fields

class SearchableQuerySet(QuerySet):
    """
    QuerySet providing main search functionality for
    ``SearchableManager``.
    """

    def __init__(self, *args, **kwargs):
        self._search_ordered = False
        self._search_terms = set()
        self._search_fields = kwargs.pop("search_fields", {})
        super(SearchableQuerySet, self).__init__(*args, **kwargs)

    def search(self, query, search_fields=None):
        """
        Build a queryset matching words in the given search query,
        treating quoted terms as exact phrases and taking into
        account + and - symbols as modifiers controlling which terms
        to require and exclude.
        """

        # ### DETERMINE FIELDS TO SEARCH ###

        # Use search_fields arg if given, otherwise use search_fields
        # initially configured by the manager class.
        if search_fields:
            self._search_fields = search_fields_to_dict(search_fields)
        if not self._search_fields:
            return self.none()

        # ### BUILD LIST OF TERMS TO SEARCH FOR ###

        # Remove extra spaces, put modifiers inside quoted terms.
        terms = " ".join(query.split()).replace("+ ", "+")     \
                                       .replace('+"', '"+')    \
                                       .replace("- ", "-")     \
                                       .replace('-"', '"-')    \
                                       .split('"')
        # Strip punctuation other than modifiers from terms and create
        # terms list, first from quoted terms and then remaining words.
        terms = [("" if t[0:1] not in "+-" else t[0:1]) + t.strip(punctuation)
            for t in terms[1::2] + "".join(terms[::2]).split()]
        # Remove stop words from terms that aren't quoted or use
        # modifiers, since words with these are an explicit part of
        # the search query. If doing so ends up with an empty term
        # list, then keep the stop words.
        terms_no_stopwords = [t for t in terms if t.lower() not in
            settings.STOP_WORDS]
        get_positive_terms = lambda terms: [t.lower().strip(punctuation)
            for t in terms if t[0:1] != "-"]
        positive_terms = get_positive_terms(terms_no_stopwords)
        if positive_terms:
            terms = terms_no_stopwords
        else:
            positive_terms = get_positive_terms(terms)
        # Append positive terms (those without the negative modifier)
        # to the internal list for sorting when results are iterated.
        if not positive_terms:
            return self.none()
        else:
            self._search_terms.update(positive_terms)

        # ### BUILD QUERYSET FILTER ###

        # Create the queryset combining each set of terms.
        excluded = [reduce(iand, [~Q(**{"%s__icontains" % f: t[1:]}) for f in
            self._search_fields.keys()]) for t in terms if t[0:1] == "-"]
        required = [reduce(ior, [Q(**{"%s__icontains" % f: t[1:]}) for f in
            self._search_fields.keys()]) for t in terms if t[0:1] == "+"]
        optional = [reduce(ior, [Q(**{"%s__icontains" % f: t}) for f in
            self._search_fields.keys()]) for t in terms if t[0:1] not in "+-"]
        queryset = self
        if excluded:
            queryset = queryset.filter(reduce(iand, excluded))
        if required:
            queryset = queryset.filter(reduce(iand, required))
        # Optional terms aren't relevant to the filter if there are
        # terms that are explicitly required.
        elif optional:
            queryset = queryset.filter(reduce(ior, optional))
        return queryset.distinct()

    def _clone(self, *args, **kwargs):
        """
        Ensure attributes are copied to subsequent queries.
        """
        for attr in ("_search_terms", "_search_fields", "_search_ordered"):
            kwargs[attr] = getattr(self, attr)
        return super(SearchableQuerySet, self)._clone(*args, **kwargs)

    def order_by(self, *field_names):
        """
        Mark the filter as being ordered if search has occurred.
        """
        if not self._search_ordered:
            self._search_ordered = len(self._search_terms) > 0
        return super(SearchableQuerySet, self).order_by(*field_names)

    def annotate_scores(self):
        """
        If search has occurred and no ordering has occurred, decorate
        each result with the number of search terms so that it can be
        sorted by the number of occurrence of terms.

        In the case of search fields that span model relationships, we
        cannot accurately match occurrences without some very
        complicated traversal code, which we won't attempt. So in this
        case, namely when there are no matches for a result (count=0),
        and search fields contain relationships (double underscores),
        we assume one match for one of the fields, and use the average
        weight of all search fields with relationships.
        """
        results = super(SearchableQuerySet, self).iterator()
        if self._search_terms and not self._search_ordered:
            results = list(results)
            for i, result in enumerate(results):
                count = 0
                related_weights = []
                for (field, weight) in self._search_fields.items():
                    if "__" in field:
                        related_weights.append(weight)
                    for term in self._search_terms:
                        field_value = getattr(result, field, None)
                        if field_value:
                            count += field_value.lower().count(term) * weight
                if not count and related_weights:
                    count = int(sum(related_weights) / len(related_weights))

                if result.publish_date:
                    age = (now() - result.publish_date).total_seconds()
                    if age > 0:
                        count = count / age**settings.SEARCH_AGE_SCALE_FACTOR

                results[i].result_count = count
            return iter(results)
        return results

class SearchableManager(Manager):
    """
    Manager providing a chainable queryset.
    Adapted from http://www.djangosnippets.org/snippets/562/
    search method supports spanning across models that subclass the
    model being used to search.
    """

    def __init__(self, *args, **kwargs):
        self._search_fields = kwargs.pop("search_fields", {})
        super(SearchableManager, self).__init__(*args, **kwargs)

    def get_search_fields(self):
        """
        Returns the search field names mapped to weights as a dict.
        Used in ``get_queryset`` below to tell ``SearchableQuerySet``
        which search fields to use. Also used by ``DisplayableAdmin``
        to populate Django admin's ``search_fields`` attribute.

        Search fields can be populated via
        ``SearchableManager.__init__``, which then get stored in
        ``SearchableManager._search_fields``, which serves as an
        approach for defining an explicit set of fields to be used.

        Alternatively and more commonly, ``search_fields`` can be
        defined on models themselves. In this case, we look at the
        model and all its base classes, and build up the search
        fields from all of those, so the search fields are implicitly
        built up from the inheritence chain.

        Finally if no search fields have been defined at all, we
        fall back to any fields that are ``CharField`` or ``TextField``
        instances.
        """
        search_fields = self._search_fields.copy()
        if not search_fields:
            for cls in reversed(self.model.__mro__):
                super_fields = getattr(cls, "search_fields", {})
                search_fields.update(search_fields_to_dict(super_fields))
        if not search_fields:
            search_fields = []
            for f in self.model._meta.get_fields():
                if isinstance(f, (CharField, TextField)):
                    search_fields.append(f.name)
            search_fields = search_fields_to_dict(search_fields)
        return search_fields

    def get_queryset(self):
        search_fields = self.get_search_fields()
        return SearchableQuerySet(self.model, search_fields=search_fields)

    def contribute_to_class(self, model, name):
        """
        Newer versions of Django explicitly prevent managers being
        accessed from abstract classes, which is behaviour the search
        API has always relied on. Here we reinstate it.
        """
        super(SearchableManager, self).contribute_to_class(model, name)
        setattr(model, name, ManagerDescriptor(self))

    def search(self, *args, **kwargs):
        """
        Proxy to queryset's search method for the manager's model and
        any models that subclass from this manager's model if the
        model is abstract.
        """
        if not settings.SEARCH_MODEL_CHOICES:
            # No choices defined - build a list of leaf models (those
            # without subclasses) that inherit from Displayable.
            models = [m for m in apps.get_models()
                      if issubclass(m, self.model)]
            parents = reduce(ior, [set(m._meta.get_parent_list())
                                   for m in models])
            models = [m for m in models if m not in parents]
        elif getattr(self.model._meta, "abstract", False):
            # When we're combining model subclasses for an abstract
            # model (eg Displayable), we only want to use models that
            # are represented by the ``SEARCH_MODEL_CHOICES`` setting.
            # Now this setting won't contain an exact list of models
            # we should use, since it can define superclass models such
            # as ``Page``, so we check the parent class list of each
            # model when determining whether a model falls within the
            # ``SEARCH_MODEL_CHOICES`` setting.
            search_choices = set()
            models = set()
            parents = set()
            errors = []
            for name in settings.SEARCH_MODEL_CHOICES:
                try:
                    model = apps.get_model(*name.split(".", 1))
                except LookupError:
                    errors.append(name)
                else:
                    search_choices.add(model)
            if errors:
                raise ImproperlyConfigured("Could not load the model(s) "
                        "%s defined in the 'SEARCH_MODEL_CHOICES' setting."
                        % ", ".join(errors))

            for model in apps.get_models():
                # Model is actually a subclasses of what we're
                # searching (eg Displayabale)
                is_subclass = issubclass(model, self.model)
                # Model satisfies the search choices list - either
                # there are no search choices, model is directly in
                # search choices, or its parent is.
                this_parents = set(model._meta.get_parent_list())
                in_choices = not search_choices or model in search_choices
                in_choices = in_choices or this_parents & search_choices
                if is_subclass and (in_choices or not search_choices):
                    # Add to models we'll seach. Also maintain a parent
                    # set, used below for further refinement of models
                    # list to search.
                    models.add(model)
                    parents.update(this_parents)
            # Strip out any models that are superclasses of models,
            # specifically the Page model which will generally be the
            # superclass for all custom content types, since if we
            # query the Page model as well, we will get duplicate
            # results.
            models -= parents
        else:
            models = [self.model]
        all_results = []
        user = kwargs.pop("for_user", None)
        for model in models:
            try:
                queryset = model.objects.published(for_user=user)
            except AttributeError:
                queryset = model.objects.get_queryset()
            all_results.extend(
                queryset.search(*args, **kwargs).annotate_scores())
        return sorted(all_results, key=lambda r: r.result_count, reverse=True)

class CurrentSiteManager(DjangoCSM):
    """
    Extends Django's site manager to first look up site by ID stored in
    the request, the session, then domain for the current request
    (accessible via threadlocals in ``mezzanine.core.request``), the
    environment variable ``MEZZANINE_SITE_ID`` (which can be used by
    management commands with the ``--site`` arg, finally falling back
    to ``settings.SITE_ID`` if none of those match a site.
    """

    use_in_migrations = False

    def __init__(self, field_name=None, *args, **kwargs):
        super(DjangoCSM, self).__init__(*args, **kwargs)
        self.__field_name = field_name
        self.__is_validated = False

    def get_queryset(self):
        if not self.__is_validated:
            self._get_field_name()
        lookup = {self.__field_name + "__id__exact": current_site_id()}
        return super(DjangoCSM, self).get_queryset().filter(**lookup)

class DisplayableManager(CurrentSiteManager, PublishedManager,
                         SearchableManager):
    """
    Manually combines ``CurrentSiteManager``, ``PublishedManager``
    and ``SearchableManager`` for the ``Displayable`` model.

    """

    def url_map(self, for_user=None, **kwargs):
        """
        Returns a dictionary of urls mapped to Displayable subclass
        instances, including a fake homepage instance if none exists.
        Used in ``mezzanine.core.sitemaps``.
        """
        class Home:
            title = _("Home")
        home = Home()
        setattr(home, "get_absolute_url", home_slug)
        items = {home.get_absolute_url(): home}
        for model in apps.get_models():
            if issubclass(model, self.model):
                for item in (model.objects.published(for_user=for_user)
                                  .filter(**kwargs)
                                  .exclude(slug__startswith="http://")
                                  .exclude(slug__startswith="https://")):
                    items[item.get_absolute_url()] = item
        return items
<commit_msg>Make the search engine work in Django 2<commit_after>from __future__ import unicode_literals

import django
from future.builtins import int, zip

from functools import reduce
from operator import ior, iand
from string import punctuation

from django.apps import apps
from django.core.exceptions import ImproperlyConfigured
from django.db.models import Manager, Q, CharField, TextField
from django.db.models.manager import ManagerDescriptor
from django.db.models.query import QuerySet
from django.contrib.sites.managers import CurrentSiteManager as DjangoCSM
from django.utils.timezone import now
from django.utils.translation import ugettext_lazy as _

from mezzanine.conf import settings
from mezzanine.utils.sites import current_site_id
from mezzanine.utils.urls import home_slug

if django.VERSION >= (1, 10):
    class ManagerDescriptor(ManagerDescriptor):
        """
        This class exists purely to skip the abstract model check
        in the __get__ method of Django's ManagerDescriptor.
        """
        def __get__(self, instance, cls=None):
            if instance is not None:
                raise AttributeError(
                    "Manager isn't accessible via %s instances" % cls.__name__
                )

            # In ManagerDescriptor.__get__, an exception is raised here
            # if cls is abstract

            if cls._meta.swapped:
                raise AttributeError(
                    "Manager isn't available; "
                    "'%s.%s' has been swapped for '%s'" % (
                        cls._meta.app_label,
                        cls._meta.object_name,
                        cls._meta.swapped,
                    )
                )

            return cls._meta.managers_map[self.manager.name]

class PublishedManager(Manager):
    """
    Provides filter for restricting items returned by status and
    publish date when the given user is not a staff member.
    """

    def published(self, for_user=None):
        """
        For non-staff users, return items with a published status and
        whose publish and expiry dates fall before and after the
        current date when specified.
        """
        from mezzanine.core.models import CONTENT_STATUS_PUBLISHED
        if for_user is not None and for_user.is_staff:
            return self.all()
        return self.filter(
            Q(publish_date__lte=now()) | Q(publish_date__isnull=True),
            Q(expiry_date__gte=now()) | Q(expiry_date__isnull=True),
            Q(status=CONTENT_STATUS_PUBLISHED))

    def get_by_natural_key(self, slug):
        return self.get(slug=slug)

def search_fields_to_dict(fields):
    """
    In ``SearchableQuerySet`` and ``SearchableManager``, search fields
    can either be a sequence, or a dict of fields mapped to weights.
    This function converts sequences to a dict mapped to even weights,
    so that we're consistently dealing with a dict of fields mapped to
    weights, eg: ("title", "content") -> {"title": 1, "content": 1}
    """
    if not fields:
        return {}
    try:
        int(list(dict(fields).values())[0])
    except (TypeError, ValueError):
        fields = dict(zip(fields, [1] * len(fields)))
    return fields

class SearchableQuerySet(QuerySet):
    """
    QuerySet providing main search functionality for
    ``SearchableManager``.
    """

    def __init__(self, *args, **kwargs):
        self._search_ordered = False
        self._search_terms = set()
        self._search_fields = kwargs.pop("search_fields", {})
        super(SearchableQuerySet, self).__init__(*args, **kwargs)

    def search(self, query, search_fields=None):
        """
        Build a queryset matching words in the given search query,
        treating quoted terms as exact phrases and taking into
        account + and - symbols as modifiers controlling which terms
        to require and exclude.
        """

        # ### DETERMINE FIELDS TO SEARCH ###

        # Use search_fields arg if given, otherwise use search_fields
        # initially configured by the manager class.
        if search_fields:
            self._search_fields = search_fields_to_dict(search_fields)
        if not self._search_fields:
            return self.none()

        # ### BUILD LIST OF TERMS TO SEARCH FOR ###

        # Remove extra spaces, put modifiers inside quoted terms.
        terms = " ".join(query.split()).replace("+ ", "+")     \
                                       .replace('+"', '"+')    \
                                       .replace("- ", "-")     \
                                       .replace('-"', '"-')    \
                                       .split('"')
        # Strip punctuation other than modifiers from terms and create
        # terms list, first from quoted terms and then remaining words.
        terms = [("" if t[0:1] not in "+-" else t[0:1]) + t.strip(punctuation)
            for t in terms[1::2] + "".join(terms[::2]).split()]
        # Remove stop words from terms that aren't quoted or use
        # modifiers, since words with these are an explicit part of
        # the search query. If doing so ends up with an empty term
        # list, then keep the stop words.
        terms_no_stopwords = [t for t in terms if t.lower() not in
            settings.STOP_WORDS]
        get_positive_terms = lambda terms: [t.lower().strip(punctuation)
            for t in terms if t[0:1] != "-"]
        positive_terms = get_positive_terms(terms_no_stopwords)
        if positive_terms:
            terms = terms_no_stopwords
        else:
            positive_terms = get_positive_terms(terms)
        # Append positive terms (those without the negative modifier)
        # to the internal list for sorting when results are iterated.
        if not positive_terms:
            return self.none()
        else:
            self._search_terms.update(positive_terms)

        # ### BUILD QUERYSET FILTER ###

        # Create the queryset combining each set of terms.
        excluded = [reduce(iand, [~Q(**{"%s__icontains" % f: t[1:]}) for f in
            self._search_fields.keys()]) for t in terms if t[0:1] == "-"]
        required = [reduce(ior, [Q(**{"%s__icontains" % f: t[1:]}) for f in
            self._search_fields.keys()]) for t in terms if t[0:1] == "+"]
        optional = [reduce(ior, [Q(**{"%s__icontains" % f: t}) for f in
            self._search_fields.keys()]) for t in terms if t[0:1] not in "+-"]
        queryset = self
        if excluded:
            queryset = queryset.filter(reduce(iand, excluded))
        if required:
            queryset = queryset.filter(reduce(iand, required))
        # Optional terms aren't relevant to the filter if there are
        # terms that are explicitly required.
        elif optional:
            queryset = queryset.filter(reduce(ior, optional))
        return queryset.distinct()

    def _clone(self):
        """
        Ensure attributes are copied to subsequent queries.
        """
        clone = super(SearchableQuerySet, self)._clone()
        clone._search_terms = self._search_terms
        clone._search_fields = self._search_fields
        clone._search_ordered = self._search_ordered
        return clone

    def order_by(self, *field_names):
        """
        Mark the filter as being ordered if search has occurred.
        """
        if not self._search_ordered:
            self._search_ordered = len(self._search_terms) > 0
        return super(SearchableQuerySet, self).order_by(*field_names)

    def annotate_scores(self):
        """
        If search has occurred and no ordering has occurred, decorate
        each result with the number of search terms so that it can be
        sorted by the number of occurrence of terms.

        In the case of search fields that span model relationships, we
        cannot accurately match occurrences without some very
        complicated traversal code, which we won't attempt. So in this
        case, namely when there are no matches for a result (count=0),
        and search fields contain relationships (double underscores),
        we assume one match for one of the fields, and use the average
        weight of all search fields with relationships.
        """
        results = super(SearchableQuerySet, self).iterator()
        if self._search_terms and not self._search_ordered:
            results = list(results)
            for i, result in enumerate(results):
                count = 0
                related_weights = []
                for (field, weight) in self._search_fields.items():
                    if "__" in field:
                        related_weights.append(weight)
                    for term in self._search_terms:
                        field_value = getattr(result, field, None)
                        if field_value:
                            count += field_value.lower().count(term) * weight
                if not count and related_weights:
                    count = int(sum(related_weights) / len(related_weights))

                if result.publish_date:
                    age = (now() - result.publish_date).total_seconds()
                    if age > 0:
                        count = count / age**settings.SEARCH_AGE_SCALE_FACTOR

                results[i].result_count = count
            return iter(results)
        return results

class SearchableManager(Manager):
    """
    Manager providing a chainable queryset.
    Adapted from http://www.djangosnippets.org/snippets/562/
    search method supports spanning across models that subclass the
    model being used to search.
    """

    def __init__(self, *args, **kwargs):
        self._search_fields = kwargs.pop("search_fields", {})
        super(SearchableManager, self).__init__(*args, **kwargs)

    def get_search_fields(self):
        """
        Returns the search field names mapped to weights as a dict.
        Used in ``get_queryset`` below to tell ``SearchableQuerySet``
        which search fields to use. Also used by ``DisplayableAdmin``
        to populate Django admin's ``search_fields`` attribute.

        Search fields can be populated via
        ``SearchableManager.__init__``, which then get stored in
        ``SearchableManager._search_fields``, which serves as an
        approach for defining an explicit set of fields to be used.

        Alternatively and more commonly, ``search_fields`` can be
        defined on models themselves. In this case, we look at the
        model and all its base classes, and build up the search
        fields from all of those, so the search fields are implicitly
        built up from the inheritence chain.

        Finally if no search fields have been defined at all, we
        fall back to any fields that are ``CharField`` or ``TextField``
        instances.
        """
        search_fields = self._search_fields.copy()
        if not search_fields:
            for cls in reversed(self.model.__mro__):
                super_fields = getattr(cls, "search_fields", {})
                search_fields.update(search_fields_to_dict(super_fields))
        if not search_fields:
            search_fields = []
            for f in self.model._meta.get_fields():
                if isinstance(f, (CharField, TextField)):
                    search_fields.append(f.name)
            search_fields = search_fields_to_dict(search_fields)
        return search_fields

    def get_queryset(self):
        search_fields = self.get_search_fields()
        return SearchableQuerySet(self.model, search_fields=search_fields)

    def contribute_to_class(self, model, name):
        """
        Newer versions of Django explicitly prevent managers being
        accessed from abstract classes, which is behaviour the search
        API has always relied on. Here we reinstate it.
        """
        super(SearchableManager, self).contribute_to_class(model, name)
        setattr(model, name, ManagerDescriptor(self))

    def search(self, *args, **kwargs):
        """
        Proxy to queryset's search method for the manager's model and
        any models that subclass from this manager's model if the
        model is abstract.
        """
        if not settings.SEARCH_MODEL_CHOICES:
            # No choices defined - build a list of leaf models (those
            # without subclasses) that inherit from Displayable.
            models = [m for m in apps.get_models()
                      if issubclass(m, self.model)]
            parents = reduce(ior, [set(m._meta.get_parent_list())
                                   for m in models])
            models = [m for m in models if m not in parents]
        elif getattr(self.model._meta, "abstract", False):
            # When we're combining model subclasses for an abstract
            # model (eg Displayable), we only want to use models that
            # are represented by the ``SEARCH_MODEL_CHOICES`` setting.
            # Now this setting won't contain an exact list of models
            # we should use, since it can define superclass models such
            # as ``Page``, so we check the parent class list of each
            # model when determining whether a model falls within the
            # ``SEARCH_MODEL_CHOICES`` setting.
            search_choices = set()
            models = set()
            parents = set()
            errors = []
            for name in settings.SEARCH_MODEL_CHOICES:
                try:
                    model = apps.get_model(*name.split(".", 1))
                except LookupError:
                    errors.append(name)
                else:
                    search_choices.add(model)
            if errors:
                raise ImproperlyConfigured("Could not load the model(s) "
                        "%s defined in the 'SEARCH_MODEL_CHOICES' setting."
                        % ", ".join(errors))

            for model in apps.get_models():
                # Model is actually a subclasses of what we're
                # searching (eg Displayabale)
                is_subclass = issubclass(model, self.model)
                # Model satisfies the search choices list - either
                # there are no search choices, model is directly in
                # search choices, or its parent is.
                this_parents = set(model._meta.get_parent_list())
                in_choices = not search_choices or model in search_choices
                in_choices = in_choices or this_parents & search_choices
                if is_subclass and (in_choices or not search_choices):
                    # Add to models we'll seach. Also maintain a parent
                    # set, used below for further refinement of models
                    # list to search.
                    models.add(model)
                    parents.update(this_parents)
            # Strip out any models that are superclasses of models,
            # specifically the Page model which will generally be the
            # superclass for all custom content types, since if we
            # query the Page model as well, we will get duplicate
            # results.
            models -= parents
        else:
            models = [self.model]
        all_results = []
        user = kwargs.pop("for_user", None)
        for model in models:
            try:
                queryset = model.objects.published(for_user=user)
            except AttributeError:
                queryset = model.objects.get_queryset()
            all_results.extend(
                queryset.search(*args, **kwargs).annotate_scores())
        return sorted(all_results, key=lambda r: r.result_count, reverse=True)

class CurrentSiteManager(DjangoCSM):
    """
    Extends Django's site manager to first look up site by ID stored in
    the request, the session, then domain for the current request
    (accessible via threadlocals in ``mezzanine.core.request``), the
    environment variable ``MEZZANINE_SITE_ID`` (which can be used by
    management commands with the ``--site`` arg, finally falling back
    to ``settings.SITE_ID`` if none of those match a site.
    """

    use_in_migrations = False

    def __init__(self, field_name=None, *args, **kwargs):
        super(DjangoCSM, self).__init__(*args, **kwargs)
        self.__field_name = field_name
        self.__is_validated = False

    def get_queryset(self):
        if not self.__is_validated:
            self._get_field_name()
        lookup = {self.__field_name + "__id__exact": current_site_id()}
        return super(DjangoCSM, self).get_queryset().filter(**lookup)

class DisplayableManager(CurrentSiteManager, PublishedManager,
                         SearchableManager):
    """
    Manually combines ``CurrentSiteManager``, ``PublishedManager``
    and ``SearchableManager`` for the ``Displayable`` model.

    """

    def url_map(self, for_user=None, **kwargs):
        """
        Returns a dictionary of urls mapped to Displayable subclass
        instances, including a fake homepage instance if none exists.
        Used in ``mezzanine.core.sitemaps``.
        """
        class Home:
            title = _("Home")
        home = Home()
        setattr(home, "get_absolute_url", home_slug)
        items = {home.get_absolute_url(): home}
        for model in apps.get_models():
            if issubclass(model, self.model):
                for item in (model.objects.published(for_user=for_user)
                                  .filter(**kwargs)
                                  .exclude(slug__startswith="http://")
                                  .exclude(slug__startswith="https://")):
                    items[item.get_absolute_url()] = item
        return items
<|endoftext|>


<commit_before># --------------------------------------------------------
# Fast R-CNN
# Copyright (c) 2015 Microsoft
# Licensed under The MIT License [see LICENSE for details]
# Written by Ross Girshick
# --------------------------------------------------------

"""Transform a roidb into a trainable roidb by adding a bunch of metadata."""

import numpy as np
from fast_rcnn.config import cfg
from fast_rcnn.bbox_transform import bbox_transform
from utils.cython_bbox import bbox_overlaps
import PIL

def prepare_roidb(imdb):
    """Enrich the imdb's roidb by adding some derived quantities that
    are useful for training. This function precomputes the maximum
    overlap, taken over ground-truth boxes, between each ROI and
    each ground-truth box. The class with maximum overlap is also
    recorded.
    """
    sizes = [PIL.Image.open(imdb.image_path_at(i)).size
             for i in xrange(imdb.num_images)]
    roidb = imdb.roidb
    for i in xrange(len(imdb.image_index)):
        roidb[i]['image'] = imdb.image_path_at(i)
        roidb[i]['width'] = sizes[i][0]
        roidb[i]['height'] = sizes[i][1]
        # need gt_overlaps as a dense array for argmax
        gt_overlaps = roidb[i]['gt_overlaps'].toarray()
        # max overlap with gt over classes (columns)
        max_overlaps = gt_overlaps.max(axis=1)
        # gt class that had the max overlap
        max_classes = gt_overlaps.argmax(axis=1)
        roidb[i]['max_classes'] = max_classes
        roidb[i]['max_overlaps'] = max_overlaps
        # sanity checks
        # max overlap of 0 => class should be zero (background)
        zero_inds = np.where(max_overlaps == 0)[0]
        assert all(max_classes[zero_inds] == 0)
        # max overlap > 0 => class should not be zero (must be a fg class)
        nonzero_inds = np.where(max_overlaps > 0)[0]
        assert all(max_classes[nonzero_inds] != 0)

def add_bbox_regression_targets(roidb):
    """Add information needed to train bounding-box regressors."""
    assert len(roidb) > 0
    assert 'max_classes' in roidb[0], 'Did you call prepare_roidb first?'

    num_images = len(roidb)
    # Infer number of classes from the number of columns in gt_overlaps
    num_classes = roidb[0]['gt_overlaps'].shape[1]
    for im_i in xrange(num_images):
        rois = roidb[im_i]['boxes']
        max_overlaps = roidb[im_i]['max_overlaps']
        max_classes = roidb[im_i]['max_classes']
        roidb[im_i]['bbox_targets'] = \
                _compute_targets(rois, max_overlaps, max_classes)

    if cfg.TRAIN.BBOX_NORMALIZE_TARGETS_PRECOMPUTED:
        # Use fixed / precomputed "means" and "stds" instead of empirical values
        means = np.tile(
                np.array(cfg.TRAIN.BBOX_NORMALIZE_MEANS), (num_classes, 1))
        stds = np.tile(
                np.array(cfg.TRAIN.BBOX_NORMALIZE_STDS), (num_classes, 1))
    else:
        # Compute values needed for means and stds
        # var(x) = E(x^2) - E(x)^2
        class_counts = np.zeros((num_classes, 1)) + cfg.EPS
        sums = np.zeros((num_classes, 4))
        squared_sums = np.zeros((num_classes, 4))
        for im_i in xrange(num_images):
            targets = roidb[im_i]['bbox_targets']
            for cls in xrange(1, num_classes):
                cls_inds = np.where(targets[:, 0] == cls)[0]
                if cls_inds.size > 0:
                    class_counts[cls] += cls_inds.size
                    sums[cls, :] += targets[cls_inds, 1:].sum(axis=0)
                    squared_sums[cls, :] += \
                            (targets[cls_inds, 1:] ** 2).sum(axis=0)

        means = sums / class_counts
        stds = np.sqrt(squared_sums / class_counts - means ** 2)

    print 'bbox target means:'
    print means
    print means[1:, :].mean(axis=0) # ignore bg class
    print 'bbox target stdevs:'
    print stds
    print stds[1:, :].mean(axis=0) # ignore bg class

    # Normalize targets
    if cfg.TRAIN.BBOX_NORMALIZE_TARGETS:
        print "Normalizing targets"
        for im_i in xrange(num_images):
            targets = roidb[im_i]['bbox_targets']
            for cls in xrange(1, num_classes):
                cls_inds = np.where(targets[:, 0] == cls)[0]
                roidb[im_i]['bbox_targets'][cls_inds, 1:] -= means[cls, :]
                roidb[im_i]['bbox_targets'][cls_inds, 1:] /= stds[cls, :]
    else:
        print "NOT normalizing targets"

    # These values will be needed for making predictions
    # (the predicts will need to be unnormalized and uncentered)
    return means.ravel(), stds.ravel()

def _compute_targets(rois, overlaps, labels):
    """Compute bounding-box regression targets for an image."""
    # Indices of ground-truth ROIs
    gt_inds = np.where(overlaps == 1)[0]
    if len(gt_inds) == 0:
        # Bail if the image has no ground-truth ROIs
        return np.zeros((rois.shape[0], 5), dtype=np.float32)
    # Indices of examples for which we try to make predictions
    ex_inds = np.where(overlaps >= cfg.TRAIN.BBOX_THRESH)[0]

    # Get IoU overlap between each ex ROI and gt ROI
    ex_gt_overlaps = bbox_overlaps(
        np.ascontiguousarray(rois[ex_inds, :], dtype=np.float),
        np.ascontiguousarray(rois[gt_inds, :], dtype=np.float))

    # Find which gt ROI each ex ROI has max overlap with:
    # this will be the ex ROI's gt target
    gt_assignment = ex_gt_overlaps.argmax(axis=1)
    gt_rois = rois[gt_inds[gt_assignment], :]
    ex_rois = rois[ex_inds, :]

    targets = np.zeros((rois.shape[0], 5), dtype=np.float32)
    targets[ex_inds, 0] = labels[ex_inds]
    targets[ex_inds, 1:] = bbox_transform(ex_rois, gt_rois)
    return targets
<commit_msg>fix minor error<commit_after># --------------------------------------------------------
# Fast R-CNN
# Copyright (c) 2015 Microsoft
# Licensed under The MIT License [see LICENSE for details]
# Written by Ross Girshick
# --------------------------------------------------------

"""Transform a roidb into a trainable roidb by adding a bunch of metadata."""

import numpy as np
from fast_rcnn.config import cfg
from fast_rcnn.bbox_transform import bbox_transform
from utils.cython_bbox import bbox_overlaps
import PIL

def prepare_roidb(imdb):
    """Enrich the imdb's roidb by adding some derived quantities that
    are useful for training. This function precomputes the maximum
    overlap, taken over ground-truth boxes, between each ROI and
    each ground-truth box. The class with maximum overlap is also
    recorded.
    """
    sizes = [PIL.Image.open(imdb.image_path_at(i)).size
             for i in xrange(imdb.num_images)]
    roidb = imdb.roidb
    for i in xrange(len(imdb.image_index)):
        roidb[i]['image'] = imdb.image_path_at(i)
        roidb[i]['width'] = sizes[i][0]
        roidb[i]['height'] = sizes[i][1]
        # need gt_overlaps as a dense array for argmax
        gt_overlaps = roidb[i]['gt_overlaps'].toarray()
        # max overlap with gt over classes (columns)
        max_overlaps = gt_overlaps.max(axis=1)
        # gt class that had the max overlap
        max_classes = gt_overlaps.argmax(axis=1)
        roidb[i]['max_classes'] = max_classes
        roidb[i]['max_overlaps'] = max_overlaps
        # sanity checks
        # max overlap of 0 => class should be zero (background)
        zero_inds = np.where(max_overlaps == 0)[0]
        assert all(max_classes[zero_inds] == 0)
        # max overlap > 0 => class should not be zero (must be a fg class)
        nonzero_inds = np.where(max_overlaps > 0)[0]
        assert all(max_classes[nonzero_inds] != 0)

def add_bbox_regression_targets(roidb):
    """Add information needed to train bounding-box regressors."""
    assert len(roidb) > 0
    assert 'max_classes' in roidb[0], 'Did you call prepare_roidb first?'

    num_images = len(roidb)
    # Infer number of classes from the number of columns in gt_overlaps
    num_classes = 2 if cfg.TRAIN.AGNOSTIC else roidb[0]['gt_overlaps'].shape[1]
    for im_i in xrange(num_images):
        rois = roidb[im_i]['boxes']
        max_overlaps = roidb[im_i]['max_overlaps']
        max_classes = roidb[im_i]['max_classes']
        roidb[im_i]['bbox_targets'] = \
                _compute_targets(rois, max_overlaps, max_classes)

    if cfg.TRAIN.BBOX_NORMALIZE_TARGETS_PRECOMPUTED:
        # Use fixed / precomputed "means" and "stds" instead of empirical values
        means = np.tile(
                np.array(cfg.TRAIN.BBOX_NORMALIZE_MEANS), (num_classes, 1))
        stds = np.tile(
                np.array(cfg.TRAIN.BBOX_NORMALIZE_STDS), (num_classes, 1))
    else:
        # Compute values needed for means and stds
        # var(x) = E(x^2) - E(x)^2
        class_counts = np.zeros((num_classes, 1)) + cfg.EPS
        sums = np.zeros((num_classes, 4))
        squared_sums = np.zeros((num_classes, 4))
        for im_i in xrange(num_images):
            targets = roidb[im_i]['bbox_targets']
            for cls in xrange(1, num_classes):
                cls_inds = np.where(targets[:, 0] == cls)[0]
                if cls_inds.size > 0:
                    class_counts[cls] += cls_inds.size
                    sums[cls, :] += targets[cls_inds, 1:].sum(axis=0)
                    squared_sums[cls, :] += \
                            (targets[cls_inds, 1:] ** 2).sum(axis=0)

        means = sums / class_counts
        stds = np.sqrt(squared_sums / class_counts - means ** 2)

    print 'bbox target means:'
    print means
    print means[1:, :].mean(axis=0) # ignore bg class
    print 'bbox target stdevs:'
    print stds
    print stds[1:, :].mean(axis=0) # ignore bg class

    # Normalize targets
    if cfg.TRAIN.BBOX_NORMALIZE_TARGETS:
        print "Normalizing targets"
        for im_i in xrange(num_images):
            targets = roidb[im_i]['bbox_targets']
            for cls in xrange(1, num_classes):
                cls_inds = np.where(targets[:, 0] == cls)[0]
                roidb[im_i]['bbox_targets'][cls_inds, 1:] -= means[cls, :]
                roidb[im_i]['bbox_targets'][cls_inds, 1:] /= stds[cls, :]
    else:
        print "NOT normalizing targets"

    # These values will be needed for making predictions
    # (the predicts will need to be unnormalized and uncentered)
    return means.ravel(), stds.ravel()

def _compute_targets(rois, overlaps, labels):
    """Compute bounding-box regression targets for an image."""
    # Indices of ground-truth ROIs
    gt_inds = np.where(overlaps == 1)[0]
    if len(gt_inds) == 0:
        # Bail if the image has no ground-truth ROIs
        return np.zeros((rois.shape[0], 5), dtype=np.float32)
    # Indices of examples for which we try to make predictions
    ex_inds = np.where(overlaps >= cfg.TRAIN.BBOX_THRESH)[0]

    # Get IoU overlap between each ex ROI and gt ROI
    ex_gt_overlaps = bbox_overlaps(
        np.ascontiguousarray(rois[ex_inds, :], dtype=np.float),
        np.ascontiguousarray(rois[gt_inds, :], dtype=np.float))

    # Find which gt ROI each ex ROI has max overlap with:
    # this will be the ex ROI's gt target
    gt_assignment = ex_gt_overlaps.argmax(axis=1)
    gt_rois = rois[gt_inds[gt_assignment], :]
    ex_rois = rois[ex_inds, :]

    targets = np.zeros((rois.shape[0], 5), dtype=np.float32)
    targets[ex_inds, 0] = labels[ex_inds]
    targets[ex_inds, 1:] = bbox_transform(ex_rois, gt_rois)
    return targets
<|endoftext|>


<commit_before>from __future__ import unicode_literals

from django.contrib import admin

from mezzanine_sermons.models import SermonSeries, Sermon, SermonFile

class SermonSeriesAdmin(admin.ModelAdmin):
    pass

class SermonFileInline(admin.StackedInline):
    model = SermonFile
    verbose_name = "File"
    verbose_name_plural = "Files"

class SermonAdmin(admin.ModelAdmin):
    search_fields = ['series', 'title', 'passage', 'preacher']
    inlines = [SermonFileInline, ]

admin.site.register(SermonSeries, SermonSeriesAdmin)
admin.site.register(Sermon, SermonAdmin)

<commit_msg>Bug fix admin search field for sermons, and add admin search field for sermon_series<commit_after>from __future__ import unicode_literals

from django.contrib import admin

from mezzanine_sermons.models import SermonSeries, Sermon, SermonFile

class SermonSeriesAdmin(admin.ModelAdmin):
    search_fields = ['title', 'bible_passage', 'description']

class SermonFileInline(admin.StackedInline):
    model = SermonFile
    verbose_name = "File"
    verbose_name_plural = "Files"

class SermonAdmin(admin.ModelAdmin):
    search_fields = ['series__title', 'series__bible_passage', 'series__description', 'title', 'passage', 'preacher']
    inlines = [SermonFileInline, ]

admin.site.register(SermonSeries, SermonSeriesAdmin)
admin.site.register(Sermon, SermonAdmin)

<|endoftext|>