<commit_before>"""
raven.handlers.logging
~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import
from __future__ import print_function

import datetime
import logging
import sys
import traceback

from raven.base import Client
from raven.utils import six
from raven.utils.encoding import to_string, to_unicode
from raven.utils.stacks import iter_stack_frames, label_from_frame
from raven.utils.encoding import to_unicode

RESERVED = ('stack', 'name', 'module', 'funcName', 'args', 'msg', 'levelno', 'exc_text', 'exc_info', 'data', 'created', 'levelname', 'msecs', 'relativeCreated', 'tags')

class SentryHandler(logging.Handler, object):
    def __init__(self, *args, **kwargs):
        client = kwargs.get('client_cls', Client)
        if len(args) == 1:
            arg = args[0]
            if isinstance(arg, six.string_types):
                self.client = client(dsn=arg)
            elif isinstance(arg, Client):
                self.client = arg
            else:
                raise ValueError('The first argument to %s must be either a Client instance or a DSN, got %r instead.' % (
                    self.__class__.__name__,
                    arg,
                ))
        elif 'client' in kwargs:
            self.client = kwargs['client']
        elif len(args) == 2 and not kwargs:
            servers, key = args
            self.client = client(servers=servers, key=key)
        else:
            self.client = client(*args, **kwargs)

        logging.Handler.__init__(self, level=kwargs.get('level', logging.NOTSET))

    def emit(self, record):
        try:
            # Beware to python3 bug (see #10805) if exc_info is (None, None, None)
            self.format(record)

            # Avoid typical config issues by overriding loggers behavior
            if record.name.startswith('sentry.errors'):
                print(to_string(record.message), sys.stderr)
                return

            return self._emit(record)
        except Exception:
            print("Top level Sentry exception caught - failed creating log record", sys.stderr)
            print(to_string(record.msg), sys.stderr)
            print(to_string(traceback.format_exc()), sys.stderr)

            try:
                self.client.captureException()
            except Exception:
                pass

    def _get_targetted_stack(self, stack):
        # we might need to traverse this multiple times, so coerce it to a list
        stack = list(stack)
        frames = []
        started = False
        last_mod = ''

        for item in stack:
            if isinstance(item, (list, tuple)):
                frame, lineno = item
            else:
                frame, lineno = item, item.f_lineno

            if not started:
                f_globals = getattr(frame, 'f_globals', {})
                module_name = f_globals.get('__name__', '')
                if ((last_mod and last_mod.startswith('logging'))
                        and not module_name.startswith('logging')):
                    started = True
                else:
                    last_mod = module_name
                    continue

            frames.append((frame, lineno))

        # We failed to find a starting point
        if not frames:
            return stack

        return frames

    def _emit(self, record, **kwargs):
        data = {}

        extra = getattr(record, 'data', None)
        if not isinstance(extra, dict):
            if extra:
                extra = {'data': extra}
            else:
                extra = {}

        for k, v in six.iteritems(vars(record)):
            if k in RESERVED:
                continue
            if k.startswith('_'):
                continue
            if '.' not in k and k not in ('culprit',):
                extra[k] = v
            else:
                data[k] = v

        stack = getattr(record, 'stack', None)
        if stack is True:
            stack = iter_stack_frames()

        if stack:
            stack = self._get_targetted_stack(stack)

        date = datetime.datetime.utcfromtimestamp(record.created)
        event_type = 'raven.events.Message'
        handler_kwargs = {
            'message': record.msg,
            'params': record.args,
            'formatted': record.message,
        }

        # If there's no exception being processed, exc_info may be a 3-tuple of None
        # http://docs.python.org/library/sys.html#sys.exc_info
        if record.exc_info and all(record.exc_info):
            # capture the standard message first so that we ensure
            # the event is recorded as an exception, in addition to having our
            # message interface attached
            handler = self.client.get_handler(event_type)
            data.update(handler.capture(**handler_kwargs))

            event_type = 'raven.events.Exception'
            handler_kwargs = {'exc_info': record.exc_info}

        # HACK: discover a culprit when we normally couldn't
        elif not (data.get('sentry.interfaces.Stacktrace') or data.get('culprit')) and (record.name or record.funcName):
            culprit = label_from_frame({'module': record.name, 'function': record.funcName})
            if culprit:
                data['culprit'] = culprit

        data['level'] = record.levelno
        data['logger'] = record.name

        if hasattr(record, 'tags'):
            kwargs['tags'] = record.tags

        kwargs.update(handler_kwargs)

        return self.client.capture(
            event_type, stack=stack, data=data,
            extra=extra, date=date, **kwargs)
<commit_msg>Removed unused to_unicode import.<commit_after>"""
raven.handlers.logging
~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import
from __future__ import print_function

import datetime
import logging
import sys
import traceback

from raven.base import Client
from raven.utils import six
from raven.utils.encoding import to_string
from raven.utils.stacks import iter_stack_frames, label_from_frame

RESERVED = ('stack', 'name', 'module', 'funcName', 'args', 'msg', 'levelno', 'exc_text', 'exc_info', 'data', 'created', 'levelname', 'msecs', 'relativeCreated', 'tags')

class SentryHandler(logging.Handler, object):
    def __init__(self, *args, **kwargs):
        client = kwargs.get('client_cls', Client)
        if len(args) == 1:
            arg = args[0]
            if isinstance(arg, six.string_types):
                self.client = client(dsn=arg)
            elif isinstance(arg, Client):
                self.client = arg
            else:
                raise ValueError('The first argument to %s must be either a Client instance or a DSN, got %r instead.' % (
                    self.__class__.__name__,
                    arg,
                ))
        elif 'client' in kwargs:
            self.client = kwargs['client']
        elif len(args) == 2 and not kwargs:
            servers, key = args
            self.client = client(servers=servers, key=key)
        else:
            self.client = client(*args, **kwargs)

        logging.Handler.__init__(self, level=kwargs.get('level', logging.NOTSET))

    def emit(self, record):
        try:
            # Beware to python3 bug (see #10805) if exc_info is (None, None, None)
            self.format(record)

            # Avoid typical config issues by overriding loggers behavior
            if record.name.startswith('sentry.errors'):
                print(to_string(record.message), sys.stderr)
                return

            return self._emit(record)
        except Exception:
            print("Top level Sentry exception caught - failed creating log record", sys.stderr)
            print(to_string(record.msg), sys.stderr)
            print(to_string(traceback.format_exc()), sys.stderr)

            try:
                self.client.captureException()
            except Exception:
                pass

    def _get_targetted_stack(self, stack):
        # we might need to traverse this multiple times, so coerce it to a list
        stack = list(stack)
        frames = []
        started = False
        last_mod = ''

        for item in stack:
            if isinstance(item, (list, tuple)):
                frame, lineno = item
            else:
                frame, lineno = item, item.f_lineno

            if not started:
                f_globals = getattr(frame, 'f_globals', {})
                module_name = f_globals.get('__name__', '')
                if ((last_mod and last_mod.startswith('logging'))
                        and not module_name.startswith('logging')):
                    started = True
                else:
                    last_mod = module_name
                    continue

            frames.append((frame, lineno))

        # We failed to find a starting point
        if not frames:
            return stack

        return frames

    def _emit(self, record, **kwargs):
        data = {}

        extra = getattr(record, 'data', None)
        if not isinstance(extra, dict):
            if extra:
                extra = {'data': extra}
            else:
                extra = {}

        for k, v in six.iteritems(vars(record)):
            if k in RESERVED:
                continue
            if k.startswith('_'):
                continue
            if '.' not in k and k not in ('culprit',):
                extra[k] = v
            else:
                data[k] = v

        stack = getattr(record, 'stack', None)
        if stack is True:
            stack = iter_stack_frames()

        if stack:
            stack = self._get_targetted_stack(stack)

        date = datetime.datetime.utcfromtimestamp(record.created)
        event_type = 'raven.events.Message'
        handler_kwargs = {
            'message': record.msg,
            'params': record.args,
            'formatted': record.message,
        }

        # If there's no exception being processed, exc_info may be a 3-tuple of None
        # http://docs.python.org/library/sys.html#sys.exc_info
        if record.exc_info and all(record.exc_info):
            # capture the standard message first so that we ensure
            # the event is recorded as an exception, in addition to having our
            # message interface attached
            handler = self.client.get_handler(event_type)
            data.update(handler.capture(**handler_kwargs))

            event_type = 'raven.events.Exception'
            handler_kwargs = {'exc_info': record.exc_info}

        # HACK: discover a culprit when we normally couldn't
        elif not (data.get('sentry.interfaces.Stacktrace') or data.get('culprit')) and (record.name or record.funcName):
            culprit = label_from_frame({'module': record.name, 'function': record.funcName})
            if culprit:
                data['culprit'] = culprit

        data['level'] = record.levelno
        data['logger'] = record.name

        if hasattr(record, 'tags'):
            kwargs['tags'] = record.tags

        kwargs.update(handler_kwargs)

        return self.client.capture(
            event_type, stack=stack, data=data,
            extra=extra, date=date, **kwargs)
<|endoftext|>


<commit_before>"""

Poll a base directory for incoming Dicom files ready for processing. We assume that
Dicom files are already processed by the hierarchize.sh script with the following directory structure:

  2016
     _ 20160407
        _ PR01471_CC082251
           _ .ready
           _ 1
              _ al_B1mapping_v2d
              _ gre_field_mapping_1acq_rl
              _ localizer

We are looking for the presence of the .ready marker file indicating that pre-processing of an MRI session is complete.

"""

import logging
import pre_process_dicom
import pprint
import os, sys

from datetime import datetime, timedelta, time
from airflow import DAG
from airflow.operators import BashOperator, PythonOperator, TriggerDagRunOperator
from airflow.models import Variable

# functions

def trigger_preprocessing(context, dagrun):
    if True:
        dagrun.payload = context['params']
        return True

# constants

#START = datetime.combine(datetime.today() - timedelta(days=2), datetime.min.time()) + timedelta(hours=10)
START = datetime.now()

DAG_NAME = 'poll_pre_process_incoming'

# Define the DAG

default_args = {
 'owner': 'airflow',
 'depends_on_past': False,
 'start_date': START,
 'retries': 1,
 'retry_delay': timedelta(seconds=120),
 'email_on_failure': True,
 'email_on_retry': True
}

dag = DAG(dag_id=DAG_NAME,
          default_args=default_args,
          schedule_interval='1 * * * *')

preprocessing_data_folder = Variable.get("preprocessing_data_folder")

scan_ready_dirs = BashOperator(
    task_id='scan_dirs_ready_for_preprocessing',
    bash_command="echo 'Scaning directories ready for processing'",
    wait_for_downstream=True,
    dag=dag)

if not os.path.exists(preprocessing_data_folder):
    os.makedirs(preprocessing_data_folder)

for fname in os.listdir(preprocessing_data_folder):
    path = os.path.join(preprocessing_data_folder, fname)
    if os.path.isdir(path):
        ready_file_marker = os.path.join(path, '.ready')
        if os.access(ready_file_marker, os.R_OK):

            logging.info('Executing: %s', str(fname))

            preprocessing_ingest = TriggerDagRunOperator(
                # need to wrap task_id in str() because log_name returns as unicode
                task_id=str('preprocess_ingest_%s' % fname),
                trigger_dag_id=preprocess_dicom.DAG_NAME,
                python_callable=trigger_preprocessing,
                params={'folder': path, 'session_id': fname},
                dag=dag
            )

            preprocessing_ingest.set_upstream(scan_ready_dirs)
<commit_msg>Walking skeleton<commit_after>"""

Poll a base directory for incoming Dicom files ready for processing. We assume that
Dicom files are already processed by the hierarchize.sh script with the following directory structure:

  2016
     _ 20160407
        _ PR01471_CC082251
           _ .ready
           _ 1
              _ al_B1mapping_v2d
              _ gre_field_mapping_1acq_rl
              _ localizer

We are looking for the presence of the .ready marker file indicating that pre-processing of an MRI session is complete.

"""

import logging
import pre_process_dicom
import pprint
import os, sys

from datetime import datetime, timedelta, time
from airflow import DAG
from airflow.operators import BashOperator, PythonOperator, TriggerDagRunOperator
from airflow.models import Variable

# functions

def trigger_preprocessing(context, dag_run_obj):
    if True:
        dag_run_obj.payload = context['params']
        return dag_run_obj

# constants

START = datetime.utcnow()
START = datetime.combine(START.date(), time(START.hour, START.minute))
#START = datetime.combine(datetime.today() - timedelta(days=2), datetime.min.time()) + timedelta(hours=10)
#START = datetime.now()

DAG_NAME = 'poll_pre_process_incoming'

# Define the DAG

default_args = {
 'owner': 'airflow',
 'depends_on_past': False,
 'start_date': START,
 'retries': 1,
 'retry_delay': timedelta(seconds=120),
 'email_on_failure': True,
 'email_on_retry': True
}

dag = DAG(dag_id=DAG_NAME,
          default_args=default_args,
          schedule_interval='0 * * * *')

preprocessing_data_folder = Variable.get("preprocessing_data_folder")

scan_ready_dirs = BashOperator(
    task_id='scan_dirs_ready_for_preprocessing',
    bash_command="echo 'Scaning directories ready for processing'",
    dag=dag)

if not os.path.exists(preprocessing_data_folder):
    os.makedirs(preprocessing_data_folder)

for fname in os.listdir(preprocessing_data_folder):
    path = os.path.join(preprocessing_data_folder, fname)
    if os.path.isdir(path):
        ready_file_marker = os.path.join(path, '.ready')
        if os.access(ready_file_marker, os.R_OK):

            logging.info('Executing: %s', str(fname))

            preprocessing_ingest = TriggerDagRunOperator(
                # need to wrap task_id in str() because log_name returns as unicode
                task_id=str('preprocess_ingest_%s' % fname),
                trigger_dag_id=pre_process_dicom.DAG_NAME,
                python_callable=trigger_preprocessing,
                params={'folder': path, 'session_id': fname},
                dag=dag
            )

            preprocessing_ingest.set_upstream(scan_ready_dirs)
<|endoftext|>


<commit_before>from contextlib import contextmanager, redirect_stdout
from sympy import Eq, Lambda, Function, Indexed, latex, Subs
from functools import partial

def define(let, be, ctor=Eq, **kwds,):

    if 'evaluate' not in kwds: kwds['evaluate'] = False # If `evaluate` is already given, use it as it is,
    # otherwise set to `False` to preevent evaluation
                                                                    # by `Eq`, which implicitly do simplifications;
    return ctor(let, be, **kwds)                                      # finally, return an equation object.

@contextmanager
def lift_to_Lambda(eq, return_eq=False, lhs_handler=lambda args: []):

    lhs = eq.lhs
    args = (lhs.args[1:] if isinstance(lhs, Indexed) else           # get arguments with respect to the type of `lhs`
            lhs.args     if isinstance(lhs, Function) else          # object; here we handle both function and
            lhs_handler(lhs))                                       # subscript notations. Finally, `Lambda` is the
    yield Lambda(args, eq if return_eq else eq.rhs)                 # class of callable objects in SymPy.

def save_latex_repr(term, filename, iterable=False):

    with open(filename, 'w') as f:
        with redirect_stdout(f):
            print('.. math::\n\n', end='')
            if iterable:
                for subterm in term:
                    print('\t& ' + r'{}\\'.format(latex(subterm)))
            else:
                print('\t' + latex(term))

def identity(*args):
    return args if len(args) > 1 else args[0]                       # Return the given arguments, simply.

def foldl1(f, lst, key=identity):

    first, *rest = lst                                              # Destructure `lst` into car and cdr.
    accumulated = key(first)                                        # Init the accumulator with the first element.
    while rest:                                                     # While there is element in the cdr,
        current, *rest = rest                                       # take the next element to combine
        accumulated = f(accumulated, key(current))                  # with the accumulated value via the function `f`.

    return accumulated                                              # Finally, return the accumulated value.

class FEq(Eq):

    def __call__(self, *args, **kwds):
        with lift_to_Lambda(self, **kwds) as feq:
            applied = feq(*args)
            if isinstance(applied, Eq):
                subs = Subs(applied, self.lhs, self.rhs)
                setattr(subs, 'substitution', partial(lambda inner_self, *args, **kwds: self, subs))
                return subs
            else:
                return applied

<commit_msg>Add some definitions to `FEq` class.<commit_after>from contextlib import contextmanager, redirect_stdout
from sympy import Eq, Lambda, Function, Indexed, latex, Subs
from functools import partial

def define(let, be, ctor=Eq, **kwds,):

    if 'evaluate' not in kwds: kwds['evaluate'] = False # If `evaluate` is already given, use it as it is,
    # otherwise set to `False` to preevent evaluation
                                                                    # by `Eq`, which implicitly do simplifications;
    return ctor(let, be, **kwds)                                      # finally, return an equation object.

@contextmanager
def lift_to_Lambda(eq, return_eq=False, lhs_handler=lambda args: []):

    lhs = eq.lhs
    args = (lhs.args[1:] if isinstance(lhs, Indexed) else           # get arguments with respect to the type of `lhs`
            lhs.args     if isinstance(lhs, Function) else          # object; here we handle both function and
            lhs_handler(lhs))                                       # subscript notations. Finally, `Lambda` is the
    yield Lambda(args, eq if return_eq else eq.rhs)                 # class of callable objects in SymPy.

def save_latex_repr(term, filename, iterable=False):

    with open(filename, 'w') as f:
        with redirect_stdout(f):
            print('.. math::\n\n', end='')
            if iterable:
                for subterm in term:
                    print('\t& ' + r'{}\\'.format(latex(subterm)))
            else:
                print('\t' + latex(term))

def identity(*args):
    return args if len(args) > 1 else args[0]                       # Return the given arguments, simply.

def foldl1(f, lst, key=identity):

    first, *rest = lst                                              # Destructure `lst` into car and cdr.
    accumulated = key(first)                                        # Init the accumulator with the first element.
    while rest:                                                     # While there is element in the cdr,
        current, *rest = rest                                       # take the next element to combine
        accumulated = f(accumulated, key(current))                  # with the accumulated value via the function `f`.

    return accumulated                                              # Finally, return the accumulated value.

class FEq(Eq):

    def __call__(self, *args, **kwds):
        with lift_to_Lambda(self, **kwds) as feq:
            applied = feq(*args)
            if isinstance(applied, Eq):
                subs = Subs(applied, self.lhs, self.rhs)
                setattr(subs, 'substitution', partial(lambda inner_self, *args, **kwds: self, subs))
                return subs
            else:
                return applied

    def swap(self):
        return self.__class__(self.rhs, self.lhs)

    def __iter__(self):
        yield self.lhs, self.rhs

    def as_substitution(self):
        return dict(self)

    def __mod__(self, arg):

        def S(term):
            return term.subs(self, simultaneous=True)

        if isinstance(arg, Eq):
            lhs = self.lhs if self.lhs == arg.lhs else S(arg.lhs)
            rhs = S(arg.rhs)
            return arg.__class__(lhs, rhs, evaluate=False)
        else:
            return S(arg)

<|endoftext|>


<commit_before>############################################################################
# Copyright (C) 2013 tele <tele@rhizomatica.org>
#
# Numbering module
# This file is part of RCCN
#
# RCCN is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# RCCN is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
############################################################################

import sys
sys.path.append("..")
from config import *
from ESL import *

class NumberingException(Exception):
    pass

class Numbering:

    def is_number_sip_connected(self, session, number):
        try:
            session.execute('set', "sofia_contact_=${sofia_contact("+number+")}")
            _sofia_contact=session.getVariable("sofia_contact_")
            if _sofia_contact == 'error/user_not_registered':
                session.execute('set', "sofia_contact_=${sofia_contact(*/"+number+"@"+wan_ip_address+")}")
                _sofia_contact=session.getVariable("sofia_contact_")
            log.info('Sofia Contact: %s' % _sofia_contact)
            if _sofia_contact == '' or _sofia_contact == 'error/user_not_registered':
                return False
            return _sofia_contact
        except Exception as ex:
            log.info('Exception: %s' % ex)

    def is_number_sip_connected_no_session(self, number):
        # Carefult with this, seems to lock up FS console if called from chatplan.
        # That is to say, don't connect back via ESL from the chatplan.
        try:
            con = ESLconnection("127.0.0.1", "8021", "ClueCon")
            e = con.api("sofia_contact "+str(number))
            _sofia_contact=e.getBody()
            if _sofia_contact == 'error/user_not_registered':
                e = con.api( "sofia_contact */" + str(number) + "@" + str(wan_ip_address) )
                _sofia_contact=e.getBody()
            log.info('Sofia Contact: %s' % _sofia_contact)
            if _sofia_contact == '' or _sofia_contact == 'error/user_not_registered':
                return False
            return _sofia_contact
        except Exception as ex:
            log.info('Exception: %s' % ex)

    def is_number_did(self, destination_number):
        try:
            cur = db_conn.cursor()
            cur.execute("SELECT phonenumber FROM dids WHERE phonenumber=%(number)s", {'number': destination_number})
            did = cur.fetchone()
            #log.debug("Value of did var: %s" % did)
            if did != None:
                return True
            else:
                return False
        except psycopg2.DatabaseError as e:
            raise NumberingException('Database error checking DID: %s' % e)

    def is_number_local(self, destination_number):
        # check if extension if yes add internal_prefix
        if len(destination_number) == 5:
            destination_number = config['internal_prefix'] + destination_number

        try:
            cur = db_conn.cursor()
            cur.execute('SELECT msisdn FROM subscribers WHERE msisdn=%(number)s', {'number': destination_number})
            dest = cur.fetchone()
            if dest != None:
                destn = dest[0]
                # check if number is local to the site
                if destn[:6] == config['internal_prefix']:
                    return True
            else:
                return False
        except psycopg2.DatabaseError as e:
            raise NumberingException('Database error checking if number is local:' % e )

    def is_number_internal(self, destination_number):
        siteprefix = destination_number[:6]
        if siteprefix == config['internal_prefix']:
            return False
        sites = riak_client.bucket('sites')
        if sites.get(siteprefix).exists == True:
            return True
        else:
            return False

    def is_number_roaming(self, number):
        try:
            cur = db_conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
            cur.execute('SELECT * FROM hlr WHERE msisdn=%(msisdn)s', {'msisdn': number})
            subscriber = cur.fetchone()
            if subscriber != None:
                if subscriber['home_bts'] != subscriber['current_bts']:
                    if subscriber['authorized'] == 1:
                        return True
                    else:
                        raise NumberingException('RK_DB subscriber %s is roaming on %s but is not authorized' % (number, subscriber['current_bts']))
            return False
        except psycopg2.DatabaseError as e:
            raise NumberingException('PG_HLR error checking if number is in roaming:' % e)

    def get_dhlr_entry(self,imsi):
        try:
            rk_hlr = riak_client.bucket('hlr')
            subscriber = rk_hlr.get(str(imsi), timeout=RIAK_TIMEOUT)
            if not subscriber.exists:
                raise NumberingException('RK_DB imsi %s not found' % imsi)
            return subscriber.data
        except riak.RiakError as e:
            raise NumberingException('RK_HLR error getting the msisdn from an imsi: %s' % e)
        except socket.error:
            raise NumberingException('RK_HLR error: unable to connect')

    def get_msisdn_from_imsi(self, imsi):
        try:
            rk_hlr = riak_client.bucket('hlr')
            subscriber = rk_hlr.get(str(imsi), timeout=RIAK_TIMEOUT)
            if not subscriber.exists:
                raise NumberingException('RK_DB imsi %s not found' % imsi)
            if subscriber.data["authorized"] != 1:
                raise NumberingException('RK_DB imsi %s (%s) not authorized' % (imsi, subscriber.data['msisdn']))
            return subscriber.data["msisdn"]

        except riak.RiakError as e:
            raise NumberingException('RK_HLR error getting the msisdn from an imsi: %s' % e)
        except socket.error:
            raise NumberingException('RK_HLR error: unable to connect')

    def get_msisdn_from_imei(self, imei):
        try:
            sq_hlr = sqlite3.connect(sq_hlr_path)
            sq_hlr_cursor = sq_hlr.cursor()
            sql=('SELECT Equipment.imei, Subscriber.imsi, '
            'Subscriber.extension, Subscriber.updated '
            'FROM Equipment, EquipmentWatch, Subscriber '
            'WHERE EquipmentWatch.equipment_id=Equipment.id '
            'AND EquipmentWatch.subscriber_id=Subscriber.id '
            'AND Equipment.imei=? '
            'ORDER BY Subscriber.updated DESC LIMIT 1;')
            print sql
            sq_hlr_cursor.execute(sql, [(imei)])
            extensions = sq_hlr_cursor.fetchall()
            sq_hlr.close()
            return extensions
        except sqlite3.Error as e:
            sq_hlr.close()
            raise NumberingException('SQ_HLR error: %s' % e.args[0])

    def get_imei_autocomplete(self, partial_imei=''):
        try:
            sq_hlr = sqlite3.connect(sq_hlr_path)
            sq_hlr_cursor = sq_hlr.cursor()
            sql='SELECT DISTINCT Equipment.imei FROM Equipment '
            if partial_imei!='':
                sql+='WHERE Equipment.imei LIKE ? ORDER BY Equipment.imei ASC'
                sq_hlr_cursor.execute(sql, [(partial_imei+'%')])
            else:
                sq_hlr_cursor.execute(sql)
            imeis = sq_hlr_cursor.fetchall()
            sq_hlr.close()
            if imeis == []:
                return []
            if len(imeis)==1:
                data=self.get_msisdn_from_imei(imeis[0][0])
                return data
            else:
                return imeis
        except sqlite3.Error as e:
            sq_hlr.close()
            raise NumberingException('SQ_HLR error: %s' % e.args[0])

    def get_local_hlr_btsinfo(self, number):
        try:
            cur = db_conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
            cur.execute('SELECT home_bts,current_bts FROM hlr WHERE msisdn=%(msisdn)s', {'msisdn': number})
            subscriber = cur.fetchone()
            if subscriber != None:
                return subscriber
            else:
                raise NumberingException('PG_DB subscriber not found: %s' % number)
            return False
        except psycopg2.DatabaseError as e:
            raise NumberingException('PG_HLR error getting bts info for:' % e)

    def get_current_bts(self, number):
        try:
            cur = db_conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
            cur.execute('SELECT current_bts FROM hlr WHERE msisdn=%(msisdn)s', {'msisdn': number})
            subscriber = cur.fetchone()
            if subscriber != None:
                return subscriber['current_bts']
            else:
                raise NumberingException('PG_DB subscriber not found: %s' % number)
            return False

        except psycopg2.DatabaseError as e:
            raise NumberingException('PG_HLR error checking if number is in roaming:' % e)

    def get_bts_distributed_hlr(self, imsi, bts):
        try:
            rk_hlr = riak_client.bucket('hlr')
            subscriber = rk_hlr.get(imsi, timeout=RIAK_TIMEOUT)
            return subscriber.data[bts]
        except riak.RiakError as e:
            raise SubscriberException('RK_HLR error: %s' % e)

    def get_site_ip(self, destination_number):
        siteprefix = destination_number[:6]
        site = riak_client.bucket('sites')
        site_data = site.get(siteprefix)
        if site_data.data['ip_address'] != None:
            return site_data.data['ip_address']
        else:
            raise NumberingException('RK_DB Error no IP found for site %s' % site)

    def get_callerid(self):
        try:
            cur = db_conn.cursor()
            # to still be decided the logic of dids
            cur.execute('select callerid from dids,providers where providers.id = dids.provider_id and providers.active = 1 order by dids.id asc limit 1')
            callerid = cur.fetchone()
            if callerid != None:
                return callerid[0]
            else:
                return None
        except psycopg2.DatabaseError as e:
            raise NumberingException('Database error getting CallerID: %s' % e)

    def get_did_subscriber(self, destination_number):
        try:
            cur = db_conn.cursor()
            cur.execute('select subscriber_number from dids where phonenumber=%(number)s', {'number': destination_number})
            dest = cur.fetchone()
            if dest != None:
                return dest[0]
        except psycopg2.DatabaseError as e:
            raise NumberingException('Database error getting subscriber number associated to the DID: %s' % e)

    def get_gateway(self):
        try:
            cur = db_conn.cursor()
            cur.execute('select provider_name from providers where active = 1')
            gw = cur.fetchone()
            if gw != None:
                return gw[0]
            else:
                return None
        except psycopg2.DatabaseError, e:
            raise NumberingException('Database error getting the Gateway: %s' % e)

if __name__ == '__main__':
	num = Numbering()
	num.is_number_roaming('66666139666')
<commit_msg>numbering: is_number_internal() Try to avoid riak<commit_after>############################################################################
# Copyright (C) 2013 tele <tele@rhizomatica.org>
#
# Numbering module
# This file is part of RCCN
#
# RCCN is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# RCCN is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
############################################################################

import sys
sys.path.append("..")
from config import *
from ESL import *

class NumberingException(Exception):
    pass

class Numbering:

    def is_number_sip_connected(self, session, number):
        try:
            session.execute('set', "sofia_contact_=${sofia_contact("+number+")}")
            _sofia_contact=session.getVariable("sofia_contact_")
            if _sofia_contact == 'error/user_not_registered':
                session.execute('set', "sofia_contact_=${sofia_contact(*/"+number+"@"+wan_ip_address+")}")
                _sofia_contact=session.getVariable("sofia_contact_")
            log.info('Sofia Contact: %s' % _sofia_contact)
            if _sofia_contact == '' or _sofia_contact == 'error/user_not_registered':
                return False
            return _sofia_contact
        except Exception as ex:
            log.info('Exception: %s' % ex)

    def is_number_sip_connected_no_session(self, number):
        # Carefult with this, seems to lock up FS console if called from chatplan.
        # That is to say, don't connect back via ESL from the chatplan.
        try:
            con = ESLconnection("127.0.0.1", "8021", "ClueCon")
            e = con.api("sofia_contact "+str(number))
            _sofia_contact=e.getBody()
            if _sofia_contact == 'error/user_not_registered':
                e = con.api( "sofia_contact */" + str(number) + "@" + str(wan_ip_address) )
                _sofia_contact=e.getBody()
            log.info('Sofia Contact: %s' % _sofia_contact)
            if _sofia_contact == '' or _sofia_contact == 'error/user_not_registered':
                return False
            return _sofia_contact
        except Exception as ex:
            log.info('Exception: %s' % ex)

    def is_number_did(self, destination_number):
        try:
            cur = db_conn.cursor()
            cur.execute("SELECT phonenumber FROM dids WHERE phonenumber=%(number)s", {'number': destination_number})
            did = cur.fetchone()
            #log.debug("Value of did var: %s" % did)
            if did != None:
                return True
            else:
                return False
        except psycopg2.DatabaseError as e:
            raise NumberingException('Database error checking DID: %s' % e)

    def is_number_local(self, destination_number):
        # check if extension if yes add internal_prefix
        if len(destination_number) == 5:
            destination_number = config['internal_prefix'] + destination_number

        try:
            cur = db_conn.cursor()
            cur.execute('SELECT msisdn FROM subscribers WHERE msisdn=%(number)s', {'number': destination_number})
            dest = cur.fetchone()
            if dest != None:
                destn = dest[0]
                # check if number is local to the site
                if destn[:6] == config['internal_prefix']:
                    return True
            else:
                return False
        except psycopg2.DatabaseError as e:
            raise NumberingException('Database error checking if number is local:' % e )

    def is_number_internal(self, destination_number):
        siteprefix = destination_number[:6]
        if siteprefix == config['internal_prefix']:
            return False
        # Try to avoid going to RIAK here
        cur = db_conn.cursor()
        cur.execute('SELECT DISTINCT home_bts FROM hlr WHERE msisdn like %(prefix)s', {'prefix': siteprefix+'%'})
        test = cur.fetchall()
        if len(test) > 1:
            log.warn('!!FIX THIS!! Got more than one home_bts from hlr for %s' % siteprefix)
            sites = riak_client.bucket('sites')
            if sites.get(siteprefix).exists == True:
                return True
            else:
                return False
        elif len(test) == 1:
            return True
        else:
            return False

    def is_number_here(self, number):
        log.info('%s %s %s %s' % (self.calling_host,mncc_ip_address,number[:6],config['internal_prefix']))
        if self.calling_host == mncc_ip_address and number[:6] == config['internal_prefix']:
            # If we are here and are a local number, then we can't be roaming
            log.warn('%s is here' % number)
            return False

    def is_number_roaming(self, number):
        try:
            cur = db_conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
            cur.execute('SELECT * FROM hlr WHERE msisdn=%(msisdn)s', {'msisdn': number})
            subscriber = cur.fetchone()
            if subscriber != None:
                if subscriber['home_bts'] != subscriber['current_bts']:
                    if subscriber['authorized'] == 1:
                        return True
                    else:
                        raise NumberingException('RK_DB subscriber %s is roaming on %s but is not authorized' % (number, subscriber['current_bts']))
            return False
        except psycopg2.DatabaseError as e:
            raise NumberingException('PG_HLR error checking if number is in roaming:' % e)

    def get_dhlr_entry(self,imsi):
        try:
            rk_hlr = riak_client.bucket('hlr')
            subscriber = rk_hlr.get(str(imsi), timeout=RIAK_TIMEOUT)
            if not subscriber.exists:
                raise NumberingException('RK_HLR imsi %s not found' % imsi)
            return subscriber.data
        except riak.RiakError as e:
            raise NumberingException('RK_HLR error getting the msisdn from an imsi: %s' % e)
        except socket.error:
            raise NumberingException('RK_HLR error: unable to connect')

    def get_msisdn_from_imsi(self, imsi):
        try:
            rk_hlr = riak_client.bucket('hlr')
            subscriber = rk_hlr.get(str(imsi), timeout=RIAK_TIMEOUT)
            if not subscriber.exists:
                raise NumberingException('RK_DB imsi %s not found' % imsi)
            if subscriber.data["authorized"] != 1:
                raise NumberingException('RK_DB imsi %s (%s) not authorized' % (imsi, subscriber.data['msisdn']))
            return subscriber.data["msisdn"]

        except riak.RiakError as e:
            raise NumberingException('RK_HLR error getting the msisdn from an imsi: %s' % e)
        except socket.error:
            raise NumberingException('RK_HLR error: unable to connect')

    def get_msisdn_from_imei(self, imei):
        try:
            sq_hlr = sqlite3.connect(sq_hlr_path)
            sq_hlr_cursor = sq_hlr.cursor()
            sql=('SELECT Equipment.imei, Subscriber.imsi, '
            'Subscriber.extension, Subscriber.updated '
            'FROM Equipment, EquipmentWatch, Subscriber '
            'WHERE EquipmentWatch.equipment_id=Equipment.id '
            'AND EquipmentWatch.subscriber_id=Subscriber.id '
            'AND Equipment.imei=? '
            'ORDER BY Subscriber.updated DESC LIMIT 1;')
            print sql
            sq_hlr_cursor.execute(sql, [(imei)])
            extensions = sq_hlr_cursor.fetchall()
            sq_hlr.close()
            return extensions
        except sqlite3.Error as e:
            sq_hlr.close()
            raise NumberingException('SQ_HLR error: %s' % e.args[0])

    def get_imei_autocomplete(self, partial_imei=''):
        try:
            sq_hlr = sqlite3.connect(sq_hlr_path)
            sq_hlr_cursor = sq_hlr.cursor()
            sql='SELECT DISTINCT Equipment.imei FROM Equipment '
            if partial_imei!='':
                sql+='WHERE Equipment.imei LIKE ? ORDER BY Equipment.imei ASC'
                sq_hlr_cursor.execute(sql, [(partial_imei+'%')])
            else:
                sq_hlr_cursor.execute(sql)
            imeis = sq_hlr_cursor.fetchall()
            sq_hlr.close()
            if imeis == []:
                return []
            if len(imeis)==1:
                data=self.get_msisdn_from_imei(imeis[0][0])
                return data
            else:
                return imeis
        except sqlite3.Error as e:
            sq_hlr.close()
            raise NumberingException('SQ_HLR error: %s' % e.args[0])

    def get_local_hlr_btsinfo(self, number):
        try:
            cur = db_conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
            cur.execute('SELECT home_bts,current_bts FROM hlr WHERE msisdn=%(msisdn)s', {'msisdn': number})
            subscriber = cur.fetchone()
            if subscriber != None:
                return subscriber
            else:
                raise NumberingException('PG_DB subscriber not found: %s' % number)
            return False
        except psycopg2.DatabaseError as e:
            raise NumberingException('PG_HLR error getting bts info for:' % e)

    def get_current_bts(self, number):
        try:
            cur = db_conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
            cur.execute('SELECT current_bts FROM hlr WHERE msisdn=%(msisdn)s', {'msisdn': number})
            subscriber = cur.fetchone()
            if subscriber != None:
                return subscriber['current_bts']
            else:
                raise NumberingException('PG_DB subscriber not found: %s' % number)
            return False

        except psycopg2.DatabaseError as e:
            raise NumberingException('PG_HLR error checking if number is in roaming:' % e)

    def get_bts_distributed_hlr(self, imsi, bts):
        try:
            rk_hlr = riak_client.bucket('hlr')
            subscriber = rk_hlr.get(imsi, timeout=RIAK_TIMEOUT)
            return subscriber.data[bts]
        except riak.RiakError as e:
            raise SubscriberException('RK_HLR error: %s' % e)

    def get_site_ip(self, destination_number):
        siteprefix = destination_number[:6]
        site = riak_client.bucket('sites')
        site_data = site.get(siteprefix)
        if site_data.data['ip_address'] != None:
            return site_data.data['ip_address']
        else:
            raise NumberingException('RK_DB Error no IP found for site %s' % site)

    def get_callerid(self):
        try:
            cur = db_conn.cursor()
            # to still be decided the logic of dids
            cur.execute('select callerid from dids,providers where providers.id = dids.provider_id and providers.active = 1 order by dids.id asc limit 1')
            callerid = cur.fetchone()
            if callerid != None:
                return callerid[0]
            else:
                return None
        except psycopg2.DatabaseError as e:
            raise NumberingException('Database error getting CallerID: %s' % e)

    def get_did_subscriber(self, destination_number):
        try:
            cur = db_conn.cursor()
            cur.execute('select subscriber_number from dids where phonenumber=%(number)s', {'number': destination_number})
            dest = cur.fetchone()
            if dest != None:
                return dest[0]
        except psycopg2.DatabaseError as e:
            raise NumberingException('Database error getting subscriber number associated to the DID: %s' % e)

    def get_gateway(self):
        try:
            cur = db_conn.cursor()
            cur.execute('select provider_name from providers where active = 1')
            gw = cur.fetchone()
            if gw != None:
                return gw[0]
            else:
                return None
        except psycopg2.DatabaseError, e:
            raise NumberingException('Database error getting the Gateway: %s' % e)

if __name__ == '__main__':
	num = Numbering()
	num.is_number_roaming('66666139666')
<|endoftext|>


<commit_before>from django.core.exceptions import ObjectDoesNotExist

from db.models.repos import CodeReference

def get_code_reference(instance, commit=None, external_repo=None):
    project = instance.project

    repo = project.repo if project.has_code else external_repo

    if not repo:
        return None

    if commit:
        try:
            return CodeReference.objects.get(repo=repo, commit=commit)
        except ObjectDoesNotExist:
            return None

    # If no commit is provided we get the last commit, and save new ref if not found
    last_commit = repo.last_commit
    if not last_commit:
        return None

    code_reference, _ = CodeReference.objects.get_or_create(repo=repo, commit=last_commit[0])
    return code_reference

def assign_code_reference(instance, commit=None):
    if instance.code_reference is not None:
        return
    build = instance.specification.build if instance.specification else None
    if not commit and build:
        commit = build.commit
    external_repo = build.git if build and build.git else None
    code_reference = get_code_reference(instance=instance,
                                        commit=commit,
                                        external_repo=external_repo)
    if code_reference:
        instance.code_reference = code_reference

    return instance
<commit_msg>Fix code_ref creation<commit_after>from django.core.exceptions import ObjectDoesNotExist

from db.models.repos import CodeReference

def get_code_reference(instance, commit=None, external_repo=None):
    project = instance.project

    if external_repo:
        repo = external_repo
    elif project.has_code:
        repo = project.repo

    if not repo:
        return None

    if commit:
        try:
            return CodeReference.objects.get(repo=repo, commit=commit)
        except ObjectDoesNotExist:
            return None

    # If no commit is provided we get the last commit, and save new ref if not found
    last_commit = repo.last_commit
    if not last_commit:
        return None

    code_reference, _ = CodeReference.objects.get_or_create(repo=repo, commit=last_commit[0])
    return code_reference

def assign_code_reference(instance, commit=None):
    if instance.code_reference is not None:
        return
    build = instance.specification.build if instance.specification else None
    if not commit and build:
        commit = build.commit
    external_repo = build.git if build and build.git else None
    code_reference = get_code_reference(instance=instance,
                                        commit=commit,
                                        external_repo=external_repo)
    if code_reference:
        instance.code_reference = code_reference

    return instance
<|endoftext|>


<commit_before># pylint: disable=redefined-builtin, wildcard-import
"""x86 specific declaration and schedules."""
from __future__ import absolute_import as _abs

from .conv2d import schedule_conv2d, schedule_conv2d_nhwc
from .binarize_pack import schedule_binarize_pack
from .binary_dense import schedule_binary_dense
from .nn import *
from .injective import *
from .pooling import schedule_pool, schedule_adaptive_pool
from .bitserial_conv2d import schedule_bitserial_conv2d
from .bitserial_dense import schedule_bitserial_dense
from .depthwise_conv2d import schedule_depthwise_conv2d_NCHWc
from .dense import _schedule_dense, _schedule_dense_pack, _schedule_dense_nopack
from .batch_matmul import schedule_batch_matmul
from .roi_align import roi_align_nchw
from .conv2d_transpose import schedule_conv2d_transpose
<commit_msg>Enable the sparse schedule (#3651)<commit_after># pylint: disable=redefined-builtin, wildcard-import
"""x86 specific declaration and schedules."""
from __future__ import absolute_import as _abs

from .conv2d import schedule_conv2d, schedule_conv2d_nhwc
from .binarize_pack import schedule_binarize_pack
from .binary_dense import schedule_binary_dense
from .nn import *
from .injective import *
from .pooling import schedule_pool, schedule_adaptive_pool
from .bitserial_conv2d import schedule_bitserial_conv2d
from .bitserial_dense import schedule_bitserial_dense
from .depthwise_conv2d import schedule_depthwise_conv2d_NCHWc
from .dense import _schedule_dense, _schedule_dense_pack, _schedule_dense_nopack
from .batch_matmul import schedule_batch_matmul
from .roi_align import roi_align_nchw
from .conv2d_transpose import schedule_conv2d_transpose
from .sparse import *
<|endoftext|>


<commit_before>from kitchen_timer import KitchenTimer
from math import floor

class PomodoroNotRunning(Exception): pass
class PomodoroAlreadyStarted(Exception): pass

def minsToSecs(mins):
    return mins * 60

class Pomodoro(object):
    '''
    A Pomodoro is the indivisable unit of time used to work on a task.
    After starting, the Pomodoro is running until:
      a) You complete it.
      b) You interrupt it.
    '''

    IDLE =          "idle"
    RUNNING =       "running"
    INTERRUPTED =   "interrupted"
    COMPLETED =     "completed"

    def __init__(self, whenTimeup, durationInMins=25):
        self._state = self.IDLE
        self._userWhenTimeup = whenTimeup
        self._durationInMins = durationInMins
        self._timer = KitchenTimer()

    def isRunning(self):
        return self._state == self.RUNNING

    def wasInterrupted(self):
        return self._state == self.INTERRUPTED

    def start(self):
        if self.isRunning():
            raise PomodoroAlreadyStarted()
        else:
            self._state = self.RUNNING
            self._timer.start(whenTimeup=self._whenTimeup, duration=minsToSecs(self._durationInMins))

    def _whenTimeup(self):
        self._state = self.COMPLETED
        if callable(self._userWhenTimeup):
            self._userWhenTimeup()

    def interrupt(self):
        if not self.isRunning():
            raise PomodoroNotRunning()
        else:
            self._timer.stop()
            self._state = self.INTERRUPTED

    @property
    def timeRemaining(self):
        if self._state == self.IDLE:
            return minsToSecs(self._durationInMins)
        else:
            return floor(self._timer.timeRemaining)

if __name__ == '__main__':
    '''
    This example starts a 1 minute Pomodoro and prints "timeup!" when it completes.
    '''
    from time import sleep, strftime, gmtime
    import sys

    def whenTimeup():
        print "timeup!"

    pp = Pomodoro(whenTimeup, durationInMins=1)
    pp.start()
    print "running Pomodoro..."
    while pp.isRunning():
        sys.stdout.write(strftime('%M:%S', gmtime(pp.timeRemaining)) + '\r')
        sys.stdout.flush()
        sleep(1)
<commit_msg>Using ceil to decrement timeRemaining since we are counting down.<commit_after>from kitchen_timer import KitchenTimer
from math import ceil

class PomodoroNotRunning(Exception): pass
class PomodoroAlreadyStarted(Exception): pass

def minsToSecs(mins):
    return mins * 60

class Pomodoro(object):
    '''
    A Pomodoro is the indivisable unit of time used to work on a task.
    After starting, the Pomodoro is running until:
      a) You complete it.
      b) You interrupt it.
    '''

    IDLE =          "idle"
    RUNNING =       "running"
    INTERRUPTED =   "interrupted"
    COMPLETED =     "completed"

    def __init__(self, whenTimeup, durationInMins=25):
        self._state = self.IDLE
        self._userWhenTimeup = whenTimeup
        self._durationInMins = durationInMins
        self._timer = KitchenTimer()

    def isRunning(self):
        return self._state == self.RUNNING

    def wasInterrupted(self):
        return self._state == self.INTERRUPTED

    def start(self):
        if self.isRunning():
            raise PomodoroAlreadyStarted()
        else:
            self._state = self.RUNNING
            self._timer.start(whenTimeup=self._whenTimeup, duration=minsToSecs(self._durationInMins))

    def _whenTimeup(self):
        self._state = self.COMPLETED
        if callable(self._userWhenTimeup):
            self._userWhenTimeup()

    def interrupt(self):
        if not self.isRunning():
            raise PomodoroNotRunning()
        else:
            self._timer.stop()
            self._state = self.INTERRUPTED

    @property
    def timeRemaining(self):
        if self._state == self.IDLE:
            return minsToSecs(self._durationInMins)
        else:
            return ceil(self._timer.timeRemaining)

if __name__ == '__main__':
    '''
    This example starts a 1 minute Pomodoro and prints "timeup!" when it completes.
    '''
    from time import sleep, strftime, gmtime
    import sys

    def whenTimeup():
        print "timeup!"

    pp = Pomodoro(whenTimeup, durationInMins=1)
    pp.start()
    print "running Pomodoro..."
    while pp.isRunning():
        sys.stdout.write(strftime('%M:%S', gmtime(pp.timeRemaining)) + '\r')
        sys.stdout.flush()
        sleep(1)
<|endoftext|>


<commit_before>'''
Importing pandasTools enables several features that allow for using RDKit molecules as columns of a Pandas dataframe.
If the dataframe is containing a molecule format in a column (e.g. smiles), like in this example:
>>> from rdkit.Chem import PandasTools
>>> import pandas as pd
>>> import os
>>> from rdkit import RDConfig
>>> antibiotics = pd.DataFrame(columns=['Name','Smiles'])
>>> antibiotics = antibiotics.append({'Smiles':'CC1(C(N2C(S1)C(C2=O)NC(=O)CC3=CC=CC=C3)C(=O)O)C','Name':'Penicilline G'}, ignore_index=True)#Penicilline G
>>> antibiotics = antibiotics.append({'Smiles':'CC1(C2CC3C(C(=O)C(=C(C3(C(=O)C2=C(C4=C1C=CC=C4O)O)O)O)C(=O)N)N(C)C)O','Name':'Tetracycline'}, ignore_index=True)#Tetracycline
>>> antibiotics = antibiotics.append({'Smiles':'CC1(C(N2C(S1)C(C2=O)NC(=O)C(C3=CC=CC=C3)N)C(=O)O)C','Name':'Ampicilline'}, ignore_index=True)#Ampicilline
>>> print antibiotics.columns
Index([Name, Smiles], dtype=object)
>>> print antibiotics
            Name                                             Smiles
0  Penicilline G    CC1(C(N2C(S1)C(C2=O)NC(=O)CC3=CC=CC=C3)C(=O)O)C
1   Tetracycline  CC1(C2CC3C(C(=O)C(=C(C3(C(=O)C2=C(C4=C1C=CC=C4...
2  Ampicilline  CC1(C(N2C(S1)C(C2=O)NC(=O)C(C3=CC=CC=C3)N)C(=O...

a new column can be created holding the respective RDKit molecule objects. The fingerprint can be included to accelerate substructure searches on the dataframe.

>>> PandasTools.AddMoleculeColumnToFrame(antibiotics,'Smiles','Molecule',includeFingerprints=True)
>>> print antibiotics.columns
Index([Name, Smiles, Molecule], dtype=object)

A substructure filter can be applied on the dataframe using the RDKit molecule column, because the ">=" operator has been modified to work as a substructure check.
Such the antibiotics containing the beta-lactam ring "C1C(=O)NC1" can be obtained by

>>> beta_lactam = Chem.MolFromSmiles('C1C(=O)NC1')
>>> beta_lactam_antibiotics = antibiotics[antibiotics['Molecule'] >= beta_lactam]
>>> print beta_lactam_antibiotics[['Name','Smiles']]
            Name                                             Smiles
0  Penicilline G    CC1(C(N2C(S1)C(C2=O)NC(=O)CC3=CC=CC=C3)C(=O)O)C
2  Ampicilline  CC1(C(N2C(S1)C(C2=O)NC(=O)C(C3=CC=CC=C3)N)C(=O...

It is also possible to load an SDF file can be load into a dataframe.

>>> sdfFile = os.path.join(RDConfig.RDDataDir,'NCI/first_200.props.sdf')
>>> frame = PandasTools.LoadSDF(sdfFile,smilesName='SMILES',molColName='Molecule',includeFingerprints=True)
>>> frame.info # doctest: +SKIP
<bound method DataFrame.info of <class 'pandas.core.frame.DataFrame'>
Int64Index: 200 entries, 0 to 199
Data columns:
AMW                       200  non-null values
CLOGP                     200  non-null values
CP                        200  non-null values
CR                        200  non-null values
DAYLIGHT.FPG              200  non-null values
DAYLIGHT_CLOGP            200  non-null values
FP                        200  non-null values
ID                        200  non-null values
ISM                       200  non-null values
LIPINSKI_VIOLATIONS       200  non-null values
NUM_HACCEPTORS            200  non-null values
NUM_HDONORS               200  non-null values
NUM_HETEROATOMS           200  non-null values
NUM_LIPINSKIHACCEPTORS    200  non-null values
NUM_LIPINSKIHDONORS       200  non-null values
NUM_RINGS                 200  non-null values
NUM_ROTATABLEBONDS        200  non-null values
P1                        30  non-null values
SMILES                    200  non-null values
Molecule                  200  non-null values
dtypes: object(20)>

In order to support rendering the molecules as images in the HTML export of the dataframe, the __str__ method is monkey-patched to return a base64 encoded PNG:
>>> molX = Chem.MolFromSmiles('Fc1cNc2ccccc12')
>>> print molX # doctest: +SKIP

This can be reverted using the ChangeMoleculeRendering method
>>> ChangeMoleculeRendering(renderer='String')
>>> print molX # doctest: +SKIP
<rdkit.Chem.rdchem.Mol object at 0x10d179440>
>>> ChangeMoleculeRendering(renderer='PNG')
>>> print molX # doctest: +SKIP

'''

from rdkit import Chem
from rdkit.Chem import Draw
from base64 import b64encode
from StringIO import StringIO
import types
try:
  import pandas as pd
  if 'display.width' in  pd.core.config._registered_options:
    pd.set_option('display.width',100000000000)
  if 'display.height' in  pd.core.config._registered_options:
    pd.set_option('display.height',100000000000)
  if 'display.max_colwidth' in  pd.core.config._registered_options:
    pd.set_option('display.max_colwidth',100000000000)
  #saves the default pandas rendering to allow restauration
  defPandasRendering = pd.core.frame.DataFrame.to_html
except ImportError:
  pd = None

def patchPandasHTMLrepr(self):
  '''
  Patched default escaping of HTML control characters to allow molecule image rendering dataframes
  '''
  formatter = pd.core.format.DataFrameFormatter(self,buf=None,columns=None,col_space=None,colSpace=None,header=True,index=True,
                                               na_rep='NaN',formatters=None,float_format=None,sparsify=None,index_names=True,
                                               justify = None, force_unicode=None,bold_rows=True,classes=None,escape=False)
  formatter.to_html()
  html = formatter.buf.getvalue()
  return html

def patchPandasHeadMethod(self,n=5):
  '''Ensure inheritance of patched to_html in "head" subframe
  '''
  df = self[:n]
  df.to_html = types.MethodType(patchPandasHTMLrepr,df)
  df.head = types.MethodType(patchPandasHeadMethod,df)
  return df

def _get_image(x):
  """displayhook function for PIL Images, rendered as PNG"""
  import pandas as pd
  sio = StringIO()
  x.save(sio,format='PNG')
  s = b64encode(sio.getvalue())
  pd.set_option('display.max_columns',len(s)+1000)
  pd.set_option('display.max_rows',len(s)+1000)
  if len(s)+100 > pd.get_option("display.max_colwidth"):
    pd.set_option("display.max_colwidth",len(s)+1000)
  return s

from rdkit import DataStructs

try:
  from rdkit.Avalon import pyAvalonTools as pyAvalonTools
  _fingerprinter=lambda x,y:pyAvalonTools.GetAvalonFP(x,isQuery=y,bitFlags=pyAvalonTools.avalonSSSBits)
except ImportError:
  _fingerprinter=lambda x,y:Chem.PatternFingerprint(x,fpSize=2048)

def _molge(x,y):
  """Allows for substructure check using the >= operator (X has substructure Y -> X >= Y) by
  monkey-patching the __ge__ function
  This has the effect that the pandas/numpy rowfilter can be used for substructure filtering (filtered = dframe[dframe['RDKitColumn'] >= SubstructureMolecule])
  """
  if x is None or y is None: return False
  if hasattr(x,'_substructfp'):
    if not hasattr(y,'_substructfp'):
      y._substructfp=_fingerprinter(y,True)
    if not DataStructs.AllProbeBitsMatch(y._substructfp,x._substructfp):
      return False
  return x.HasSubstructMatch(y)

Chem.Mol.__ge__ = _molge # lambda x,y: x.HasSubstructMatch(y)

def PrintAsBase64PNGString(x,renderer = None):
  '''returns the molecules as base64 encoded PNG image
  '''
  return ''%_get_image(Draw.MolToImage(x))

def PrintDefaultMolRep(x):
  return str(x.__repr__())

#Chem.Mol.__str__ = lambda x: ''%get_image(Draw.MolToImage(x))
Chem.Mol.__str__ = PrintAsBase64PNGString

def _MolPlusFingerprintFromSmiles(smi):
  '''Precomputes fingerprints and stores results in molecule objects to accelerate substructure matching
  '''
  m = Chem.MolFromSmiles(smi)
  if m is not None:
    m._substructfp=_fingerprinter(m,False)
  return m

def RenderImagesInAllDataFrames(images=True):
  '''Changes the default dataframe rendering to not escape HTML characters, thus allowing rendered images in all dataframes.
  IMPORTANT: THIS IS A GLOBAL CHANGE THAT WILL AFFECT TO COMPLETE PYTHON SESSION. If you want to change the rendering only
  for a single dataframe use the "ChangeMoleculeRendering" method instead.
  '''
  if images:
    pd.core.frame.DataFrame.to_html = patchPandasHTMLrepr
  else:
    pd.core.frame.DataFrame.to_html = defPandasRendering

def AddMoleculeColumnToFrame(frame, smilesCol='Smiles', molCol = 'ROMol',includeFingerprints=False):
  '''Converts the molecules contains in "smilesCol" to RDKit molecules and appends them to the dataframe "frame" using the specified column name.
  If desired, a fingerprint can be computed and stored with the molecule objects to accelerate substructure matching
  '''
  if not includeFingerprints:
    frame[molCol]=frame.apply(lambda x: Chem.MolFromSmiles(x[smilesCol]), axis=1)
  else:
    frame[molCol]=frame.apply(lambda x: _MolPlusFingerprintFromSmiles(x[smilesCol]), axis=1)
  print "Patching pandas"
  RenderImagesInAllDataFrames(images=True)
  #frame.to_html = types.MethodType(patchPandasHTMLrepr,frame)
  #frame.head = types.MethodType(patchPandasHeadMethod,frame)

def ChangeMoleculeRendering(frame=None, renderer='PNG'):
  '''Allows to change the rendering of the molecules between base64 PNG images and string representations.
  This serves two purposes: First it allows to avoid the generation of images if this is not desired and, secondly, it allows to enable image rendering for
  newly created dataframe that already contains molecules, without having to rerun the time-consuming AddMoleculeColumnToFrame. Note: this behaviour is, because some pandas methods, e.g. head()
  returns a new dataframe instance that uses the default pandas rendering (thus not drawing images for molecules) instead of the monkey-patched one.
  '''
  if renderer == 'String':
	Chem.Mol.__str__ = PrintDefaultMolRep
  else:
	Chem.Mol.__str__ = PrintAsBase64PNGString
  if frame is not None:
	frame.to_html = types.MethodType(patchPandasHTMLrepr,frame)

def LoadSDF(filename, smilesName='SMILES', idName='ID',molColName = 'ROMol',includeFingerprints=False):
  """ Read file in SDF format and return as Panda data frame """
  df = None
  with open(filename, 'rU') as f:
    for i, mol in enumerate(Chem.ForwardSDMolSupplier(f)):
      if mol is None: continue
      row = dict((k, mol.GetProp(k)) for k in mol.GetPropNames())
      if mol.HasProp('_Name'): row[idName] = mol.GetProp('_Name')
      row[smilesName] = Chem.MolToSmiles(mol)
      row = pd.DataFrame(row, index=[i])
      if df is None:
        df = row
      else:
        df = df.append(row)
  AddMoleculeColumnToFrame(df, smilesCol=smilesName, molCol = molColName,includeFingerprints=includeFingerprints)
  return df

if __name__ == "__main__":
  import sys
  if pd is None:
    print >>sys.stderr,"pandas installation not found, skipping tests"
  else:
    v = pd.version.version.split('.')
    if v[0]=='0' and int(v[1])<10:
      print >>sys.stderr,"pandas installation >=0.10 not found, skipping tests"
    else:
      import doctest
      doctest.testmod(optionflags=doctest.ELLIPSIS+doctest.NORMALIZE_WHITESPACE)

# $Id$
#
#  Copyright (c) 2013, Novartis Institutes for BioMedical Research Inc.
#  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#     * Neither the name of Novartis Institutes for BioMedical Research Inc.
#       nor the names of its contributors may be used to endorse or promote
#       products derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
<commit_msg>Fixes #75<commit_after>'''
Importing pandasTools enables several features that allow for using RDKit molecules as columns of a Pandas dataframe.
If the dataframe is containing a molecule format in a column (e.g. smiles), like in this example:
>>> from rdkit.Chem import PandasTools
>>> import pandas as pd
>>> import os
>>> from rdkit import RDConfig
>>> antibiotics = pd.DataFrame(columns=['Name','Smiles'])
>>> antibiotics = antibiotics.append({'Smiles':'CC1(C(N2C(S1)C(C2=O)NC(=O)CC3=CC=CC=C3)C(=O)O)C','Name':'Penicilline G'}, ignore_index=True)#Penicilline G
>>> antibiotics = antibiotics.append({'Smiles':'CC1(C2CC3C(C(=O)C(=C(C3(C(=O)C2=C(C4=C1C=CC=C4O)O)O)O)C(=O)N)N(C)C)O','Name':'Tetracycline'}, ignore_index=True)#Tetracycline
>>> antibiotics = antibiotics.append({'Smiles':'CC1(C(N2C(S1)C(C2=O)NC(=O)C(C3=CC=CC=C3)N)C(=O)O)C','Name':'Ampicilline'}, ignore_index=True)#Ampicilline
>>> print antibiotics.columns
Index([Name, Smiles], dtype=object)
>>> print antibiotics
            Name                                             Smiles
0  Penicilline G    CC1(C(N2C(S1)C(C2=O)NC(=O)CC3=CC=CC=C3)C(=O)O)C
1   Tetracycline  CC1(C2CC3C(C(=O)C(=C(C3(C(=O)C2=C(C4=C1C=CC=C4...
2  Ampicilline  CC1(C(N2C(S1)C(C2=O)NC(=O)C(C3=CC=CC=C3)N)C(=O...

a new column can be created holding the respective RDKit molecule objects. The fingerprint can be included to accelerate substructure searches on the dataframe.

>>> PandasTools.AddMoleculeColumnToFrame(antibiotics,'Smiles','Molecule',includeFingerprints=True)
>>> print antibiotics.columns
Index([Name, Smiles, Molecule], dtype=object)

A substructure filter can be applied on the dataframe using the RDKit molecule column, because the ">=" operator has been modified to work as a substructure check.
Such the antibiotics containing the beta-lactam ring "C1C(=O)NC1" can be obtained by

>>> beta_lactam = Chem.MolFromSmiles('C1C(=O)NC1')
>>> beta_lactam_antibiotics = antibiotics[antibiotics['Molecule'] >= beta_lactam]
>>> print beta_lactam_antibiotics[['Name','Smiles']]
            Name                                             Smiles
0  Penicilline G    CC1(C(N2C(S1)C(C2=O)NC(=O)CC3=CC=CC=C3)C(=O)O)C
2  Ampicilline  CC1(C(N2C(S1)C(C2=O)NC(=O)C(C3=CC=CC=C3)N)C(=O...

It is also possible to load an SDF file can be load into a dataframe.

>>> sdfFile = os.path.join(RDConfig.RDDataDir,'NCI/first_200.props.sdf')
>>> frame = PandasTools.LoadSDF(sdfFile,smilesName='SMILES',molColName='Molecule',includeFingerprints=True)
>>> frame.info # doctest: +SKIP
<bound method DataFrame.info of <class 'pandas.core.frame.DataFrame'>
Int64Index: 200 entries, 0 to 199
Data columns:
AMW                       200  non-null values
CLOGP                     200  non-null values
CP                        200  non-null values
CR                        200  non-null values
DAYLIGHT.FPG              200  non-null values
DAYLIGHT_CLOGP            200  non-null values
FP                        200  non-null values
ID                        200  non-null values
ISM                       200  non-null values
LIPINSKI_VIOLATIONS       200  non-null values
NUM_HACCEPTORS            200  non-null values
NUM_HDONORS               200  non-null values
NUM_HETEROATOMS           200  non-null values
NUM_LIPINSKIHACCEPTORS    200  non-null values
NUM_LIPINSKIHDONORS       200  non-null values
NUM_RINGS                 200  non-null values
NUM_ROTATABLEBONDS        200  non-null values
P1                        30  non-null values
SMILES                    200  non-null values
Molecule                  200  non-null values
dtypes: object(20)>

In order to support rendering the molecules as images in the HTML export of the dataframe, the __str__ method is monkey-patched to return a base64 encoded PNG:
>>> molX = Chem.MolFromSmiles('Fc1cNc2ccccc12')
>>> print molX # doctest: +SKIP

This can be reverted using the ChangeMoleculeRendering method
>>> ChangeMoleculeRendering(renderer='String')
>>> print molX # doctest: +SKIP
<rdkit.Chem.rdchem.Mol object at 0x10d179440>
>>> ChangeMoleculeRendering(renderer='PNG')
>>> print molX # doctest: +SKIP

'''

from rdkit import Chem
from rdkit.Chem import Draw
from base64 import b64encode
from StringIO import StringIO
import types
try:
  import pandas as pd
  if 'display.width' in  pd.core.config._registered_options:
    pd.set_option('display.width',100000000000)
  if 'display.height' in  pd.core.config._registered_options:
    pd.set_option('display.height',100000000000)
  if 'display.max_colwidth' in  pd.core.config._registered_options:
    pd.set_option('display.max_colwidth',100000000000)
  #saves the default pandas rendering to allow restauration
  defPandasRendering = pd.core.frame.DataFrame.to_html
except ImportError:
  pd = None

def patchPandasHTMLrepr(self):
  '''
  Patched default escaping of HTML control characters to allow molecule image rendering dataframes
  '''
  formatter = pd.core.format.DataFrameFormatter(self,buf=None,columns=None,col_space=None,colSpace=None,header=True,index=True,
                                               na_rep='NaN',formatters=None,float_format=None,sparsify=None,index_names=True,
                                               justify = None, force_unicode=None,bold_rows=True,classes=None,escape=False)
  formatter.to_html()
  html = formatter.buf.getvalue()
  return html

def patchPandasHeadMethod(self,n=5):
  '''Ensure inheritance of patched to_html in "head" subframe
  '''
  df = self[:n]
  df.to_html = types.MethodType(patchPandasHTMLrepr,df)
  df.head = types.MethodType(patchPandasHeadMethod,df)
  return df

def _get_image(x):
  """displayhook function for PIL Images, rendered as PNG"""
  import pandas as pd
  sio = StringIO()
  x.save(sio,format='PNG')
  s = b64encode(sio.getvalue())
  pd.set_option('display.max_columns',len(s)+1000)
  pd.set_option('display.max_rows',len(s)+1000)
  if len(s)+100 > pd.get_option("display.max_colwidth"):
    pd.set_option("display.max_colwidth",len(s)+1000)
  return s

from rdkit import DataStructs

try:
  from rdkit.Avalon import pyAvalonTools as pyAvalonTools
  _fingerprinter=lambda x,y:pyAvalonTools.GetAvalonFP(x,isQuery=y,bitFlags=pyAvalonTools.avalonSSSBits)
except ImportError:
  _fingerprinter=lambda x,y:Chem.PatternFingerprint(x,fpSize=2048)

def _molge(x,y):
  """Allows for substructure check using the >= operator (X has substructure Y -> X >= Y) by
  monkey-patching the __ge__ function
  This has the effect that the pandas/numpy rowfilter can be used for substructure filtering (filtered = dframe[dframe['RDKitColumn'] >= SubstructureMolecule])
  """
  if x is None or y is None: return False
  if hasattr(x,'_substructfp'):
    if not hasattr(y,'_substructfp'):
      y._substructfp=_fingerprinter(y,True)
    if not DataStructs.AllProbeBitsMatch(y._substructfp,x._substructfp):
      return False
  return x.HasSubstructMatch(y)

Chem.Mol.__ge__ = _molge # lambda x,y: x.HasSubstructMatch(y)

def PrintAsBase64PNGString(x,renderer = None):
  '''returns the molecules as base64 encoded PNG image
  '''
  return ''%_get_image(Draw.MolToImage(x))

def PrintDefaultMolRep(x):
  return str(x.__repr__())

#Chem.Mol.__str__ = lambda x: ''%get_image(Draw.MolToImage(x))
Chem.Mol.__str__ = PrintAsBase64PNGString

def _MolPlusFingerprintFromSmiles(smi):
  '''Precomputes fingerprints and stores results in molecule objects to accelerate substructure matching
  '''
  m = Chem.MolFromSmiles(smi)
  if m is not None:
    m._substructfp=_fingerprinter(m,False)
  return m

def RenderImagesInAllDataFrames(images=True):
  '''Changes the default dataframe rendering to not escape HTML characters, thus allowing rendered images in all dataframes.
  IMPORTANT: THIS IS A GLOBAL CHANGE THAT WILL AFFECT TO COMPLETE PYTHON SESSION. If you want to change the rendering only
  for a single dataframe use the "ChangeMoleculeRendering" method instead.
  '''
  if images:
    pd.core.frame.DataFrame.to_html = patchPandasHTMLrepr
  else:
    pd.core.frame.DataFrame.to_html = defPandasRendering

def AddMoleculeColumnToFrame(frame, smilesCol='Smiles', molCol = 'ROMol',includeFingerprints=False):
  '''Converts the molecules contains in "smilesCol" to RDKit molecules and appends them to the dataframe "frame" using the specified column name.
  If desired, a fingerprint can be computed and stored with the molecule objects to accelerate substructure matching
  '''
  if not includeFingerprints:
    frame[molCol]=frame.apply(lambda x: Chem.MolFromSmiles(x[smilesCol]), axis=1)
  else:
    frame[molCol]=frame.apply(lambda x: _MolPlusFingerprintFromSmiles(x[smilesCol]), axis=1)
  RenderImagesInAllDataFrames(images=True)
  #frame.to_html = types.MethodType(patchPandasHTMLrepr,frame)
  #frame.head = types.MethodType(patchPandasHeadMethod,frame)

def ChangeMoleculeRendering(frame=None, renderer='PNG'):
  '''Allows to change the rendering of the molecules between base64 PNG images and string representations.
  This serves two purposes: First it allows to avoid the generation of images if this is not desired and, secondly, it allows to enable image rendering for
  newly created dataframe that already contains molecules, without having to rerun the time-consuming AddMoleculeColumnToFrame. Note: this behaviour is, because some pandas methods, e.g. head()
  returns a new dataframe instance that uses the default pandas rendering (thus not drawing images for molecules) instead of the monkey-patched one.
  '''
  if renderer == 'String':
	Chem.Mol.__str__ = PrintDefaultMolRep
  else:
	Chem.Mol.__str__ = PrintAsBase64PNGString
  if frame is not None:
	frame.to_html = types.MethodType(patchPandasHTMLrepr,frame)

def LoadSDF(filename, smilesName='SMILES', idName='ID',molColName = 'ROMol',includeFingerprints=False):
  """ Read file in SDF format and return as Panda data frame """
  df = None
  with open(filename, 'rU') as f:
    for i, mol in enumerate(Chem.ForwardSDMolSupplier(f)):
      if mol is None: continue
      row = dict((k, mol.GetProp(k)) for k in mol.GetPropNames())
      if mol.HasProp('_Name'): row[idName] = mol.GetProp('_Name')
      row[smilesName] = Chem.MolToSmiles(mol)
      row = pd.DataFrame(row, index=[i])
      if df is None:
        df = row
      else:
        df = df.append(row)
  AddMoleculeColumnToFrame(df, smilesCol=smilesName, molCol = molColName,includeFingerprints=includeFingerprints)
  return df

if __name__ == "__main__":
  import sys
  if pd is None:
    print >>sys.stderr,"pandas installation not found, skipping tests"
  else:
    v = pd.version.version.split('.')
    if v[0]=='0' and int(v[1])<10:
      print >>sys.stderr,"pandas installation >=0.10 not found, skipping tests"
    else:
      import doctest
      failed,tried=doctest.testmod(optionflags=doctest.ELLIPSIS+doctest.NORMALIZE_WHITESPACE)
      if failed:
        sys.exit(failed)

# $Id$
#
#  Copyright (c) 2013, Novartis Institutes for BioMedical Research Inc.
#  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#     * Neither the name of Novartis Institutes for BioMedical Research Inc.
#       nor the names of its contributors may be used to endorse or promote
#       products derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Usage:
          pontoon image list [--with-ids]
          pontoon image oses
          pontoon image show <name>

Options:
    --with-ids      Include ids in output. Useful for other software that uses
                    Digital Ocean ids for input (like Packer).
    -h --help       Show this page.
"""

from docopt import docopt
from ..lib import Manager, Image
from .. import configure, ui
from ..command import Command
from .. import MOCK

class ImageCommand(Command):

    def __init__(self, config, args):
        self.config = config
        self.args = args
        self.manager = Manager(token=config['api_token'], mocked=MOCK)

    def _get_image(self, name):
        resources = self.manager.get_all_images()
        resource = [res for res in resources if res.name == name]

        if len(resource) > 1:
            ui.warning("Warning: multiple images with identical "
                       "names found. Actions on those images "
                       "will fail until this is resolved in the web UI.")
            raise Exception("Multiple images named '%s'" % name)

        if len(resource) == 0:
            raise Exception("No image named '%s'" % name)

        return resource[0]

    def list(self):
        available = self.manager.get_all_images()
        ui.message("Available images:")
        if self.args['--with-ids']:
            ui.message("   %-10s %-10s %-45s %s" % (
                       "id", "distro", "name", "slug"))
            ui.line(length=80)
        else:
            ui.message("   %-10s %-45s %s" % ("distro", "name", "slug"))
            ui.line(length=80)

        for s in available:
            if self.args['--with-ids']:
                name = s.name[:27] + "..." if len(s.name) > 30 else s.name
                ui.message(" - %-10s %-10s %-35s %s" % (
                       str(s.id) + ':', s.distribution, name, s.slug))
            else:
                name = s.name[:37] + "..." if len(s.name) > 40 else s.name
                ui.message(" - %-10s %-45s %s" % (
                           s.distribution, name, s.slug))
        return 0

    def show(self):
        image = self._get_image(self.args['<name>'])
        details = ui.format_item(image)
        ui.yaml_message(details)

    def oses(self):
        available = self.manager.get_all_images()
        ui.message("Available Operating Systems:")
        oses = []
        for image in available:
            if image.distribution not in oses:
                oses.append(image.distribution)
        for os in oses:
            ui.message(" - %s" % os)
        return 0

def main():
    try:
        configure.logger()

        config = configure.combined()

        args = docopt(str(__doc__))

        exit(ImageCommand(config, args).run())

    except Exception as e:
        ui.message(str(e))
        exit(1)

if __name__ == '__main__':
    main()
<commit_msg>Fix PEP8 E126<commit_after>#!/usr/bin/env python

"""Usage:
          pontoon image list [--with-ids]
          pontoon image oses
          pontoon image show <name>

Options:
    --with-ids      Include ids in output. Useful for other software that uses
                    Digital Ocean ids for input (like Packer).
    -h --help       Show this page.
"""

from docopt import docopt
from ..lib import Manager, Image
from .. import configure, ui
from ..command import Command
from .. import MOCK

class ImageCommand(Command):

    def __init__(self, config, args):
        self.config = config
        self.args = args
        self.manager = Manager(token=config['api_token'], mocked=MOCK)

    def _get_image(self, name):
        resources = self.manager.get_all_images()
        resource = [res for res in resources if res.name == name]

        if len(resource) > 1:
            ui.warning("Warning: multiple images with identical "
                       "names found. Actions on those images "
                       "will fail until this is resolved in the web UI.")
            raise Exception("Multiple images named '%s'" % name)

        if len(resource) == 0:
            raise Exception("No image named '%s'" % name)

        return resource[0]

    def list(self):
        available = self.manager.get_all_images()
        ui.message("Available images:")
        if self.args['--with-ids']:
            ui.message("   %-10s %-10s %-45s %s" % (
                       "id", "distro", "name", "slug"))
            ui.line(length=80)
        else:
            ui.message("   %-10s %-45s %s" % ("distro", "name", "slug"))
            ui.line(length=80)

        for s in available:
            if self.args['--with-ids']:
                name = s.name[:27] + "..." if len(s.name) > 30 else s.name
                ui.message(" - %-10s %-10s %-35s %s" % (
                           str(s.id) + ':', s.distribution, name, s.slug))
            else:
                name = s.name[:37] + "..." if len(s.name) > 40 else s.name
                ui.message(" - %-10s %-45s %s" % (
                           s.distribution, name, s.slug))
        return 0

    def show(self):
        image = self._get_image(self.args['<name>'])
        details = ui.format_item(image)
        ui.yaml_message(details)

    def oses(self):
        available = self.manager.get_all_images()
        ui.message("Available Operating Systems:")
        oses = []
        for image in available:
            if image.distribution not in oses:
                oses.append(image.distribution)
        for os in oses:
            ui.message(" - %s" % os)
        return 0

def main():
    try:
        configure.logger()

        config = configure.combined()

        args = docopt(str(__doc__))

        exit(ImageCommand(config, args).run())

    except Exception as e:
        ui.message(str(e))
        exit(1)

if __name__ == '__main__':
    main()
<|endoftext|>