<commit_before>"""
Functions and classes to simplify dealing with paths.

For portable code:
  - specify paths in unix (rather than Windows) style;
  - use resolve_path() for paths to existing files to be read,
    and normalize_path() for paths to new files to be written.

The location in which new files are created by default can be
controlled by the output_path module attribute, which applies
whenever a particular location is not specified explicitly.

$Id$
"""
__version__='$Revision$'

import os.path
import sys

from .. import param

# copied from scipy/weave/catalog.py (scipy svn 5552)
import socket,tempfile
def is_writable(dir):
    """Determine whether a given directory is writable in a portable manner.

    :Parameters:
     - dir: string
       A string represeting a path to a directory on the filesystem.

    :Returns:
      True or False.
    """

    # Do NOT use a hardcoded name here due to the danger from race conditions
    # on NFS when multiple processes are accessing the same base directory in
    # parallel.  We use both hostname and pocess id for the prefix in an
    # attempt to ensure that there can really be no name collisions (tempfile
    # appends 6 random chars to this prefix).
    prefix = 'dummy_%s_%s_' % (socket.gethostname(),os.getpid())
    try:
        tmp = tempfile.TemporaryFile(prefix=prefix,dir=dir)
    except OSError:
        return False
    # The underlying file is destroyed upon closing the file object (under
    # *nix, it was unlinked at creation time)
    tmp.close()
    return True

class Filename(param.Parameter):
    """
    Filename is a Parameter that can be set to a string specifying the
    path of a file (in unix style), and returns it in the format of
    the user's operating system.  Additionally, the specified path can
    be absolute or relative to:

    * any of the paths specified in the search_paths attribute;

    * any of the paths searched by resolve_path() (see doc for that
      function).
    """
    __slots__ = ['search_paths']

    def __init__(self,default=None,search_paths=[],**params):
        self.search_paths = search_paths
        super(Filename,self).__init__(default,**params)

    def __set__(self,obj,val):
        """
        Call Parameter's __set__, but warn if the file cannot be found.
        """
        try:
            resolve_path(val,self.search_paths)
        except IOError, e:
            param.Parameterized(name="%s.%s"%(obj.name,self._attrib_name)).warning('%s'%(e.args[0]))

        super(Filename,self).__set__(obj,val)

    def __get__(self,obj,objtype):
        """
        Return an absolute, normalized path (see resolve_path).
        """
        raw_path = super(Filename,self).__get__(obj,objtype)
        return resolve_path(raw_path,self.search_paths)

    def __getstate__(self):
        # don't want to pickle the search_paths
        state = super(Filename,self).__getstate__()
        # CBALERT: uncommenting gives an error on make tests (testsnapshots.py).
        # Testsnapshots.py runs fine on its own, and the snapshot-tests pass.
        # Must be a test interaction? Needs investigating.
        #del state['search_paths']
        return state

import topo
package_path = os.path.split(topo.__file__)[0]

# CEBALERT: we should try to remove this as soon as possible; anything
# relying on the location of the 'topographica' script should be
# re-thought (removed, or changed to output_path or package_path).
application_path = os.path.split(os.path.split(sys.executable)[0])[0]

# Location in which to create files; defaults to application_path.  If
# that's not writable, uses ~/topographica (creating it if necessary).
if is_writable(application_path):
    output_path = application_path
else:
    home_topographica = os.path.join(os.path.expanduser("~"),'topographica')
    if not os.path.exists(home_topographica):
        os.mkdir(home_topographica)
    output_path = home_topographica

def resolve_path(path,search_paths=None):
    """
    Find the path to an existing file, searching in the specified
    search paths if the filename is not absolute, and converting a
    UNIX-style path to the current OS's format if necessary.

    To turn a supplied relative path into an absolute one, the path is
    appended to each path in (search_paths+the current working
    directory+the application's base path), in that order, until the
    file is found.

    (Similar to Python's os.path.abspath(), except more search paths
    than just os.getcwd() can be used, and the file must exist.)

    An IOError is raised if the file is not found anywhere.
    """
    path = os.path.normpath(path)

    if os.path.isabs(path):
        if os.path.isfile(path):
            return path
        else:
            raise IOError('File "%s" not found.'%path)
    else:
        all_search_paths = search_paths or [] + [os.getcwd()] + [output_path] + [package_path] + [application_path]

        paths_tried = []
        for prefix in set(all_search_paths): # CEBALERT: set() loses order
            try_path = os.path.join(os.path.normpath(prefix),path)
            if os.path.isfile(try_path): return try_path
            paths_tried.append(try_path)

        raise IOError('File "'+os.path.split(path)[1]+'" was not found in the following place(s): '+str(paths_tried)+'.')

def normalize_path(path="",prefix=None):
    """
    Convert a UNIX-style path to the current OS's format,
    typically for creating a new file or directory.

    If the path is not already absolute, it will be made
    absolute (using the specified prefix, which defaults
    to filepath.output_path) in the process.

    (Should do the same as Python's os.path.abspath(), except
    using the specified prefix rather than os.getcwd().)
    """
    if not prefix:
        prefix = output_path

    if not os.path.isabs(path):
        path = os.path.join(os.path.normpath(prefix),path)

    return os.path.normpath(path)
<commit_msg>Removed erroneous use of set() (search paths were not being searched in order). It doesn't matter if a path appears more than once.<commit_after>"""
Functions and classes to simplify dealing with paths.

For portable code:
  - specify paths in unix (rather than Windows) style;
  - use resolve_path() for paths to existing files to be read,
    and normalize_path() for paths to new files to be written.

The location in which new files are created by default can be
controlled by the output_path module attribute, which applies
whenever a particular location is not specified explicitly.

$Id$
"""
__version__='$Revision$'

import os.path
import sys

from .. import param

# copied from scipy/weave/catalog.py (scipy svn 5552)
import socket,tempfile
def is_writable(dir):
    """Determine whether a given directory is writable in a portable manner.

    :Parameters:
     - dir: string
       A string represeting a path to a directory on the filesystem.

    :Returns:
      True or False.
    """

    # Do NOT use a hardcoded name here due to the danger from race conditions
    # on NFS when multiple processes are accessing the same base directory in
    # parallel.  We use both hostname and pocess id for the prefix in an
    # attempt to ensure that there can really be no name collisions (tempfile
    # appends 6 random chars to this prefix).
    prefix = 'dummy_%s_%s_' % (socket.gethostname(),os.getpid())
    try:
        tmp = tempfile.TemporaryFile(prefix=prefix,dir=dir)
    except OSError:
        return False
    # The underlying file is destroyed upon closing the file object (under
    # *nix, it was unlinked at creation time)
    tmp.close()
    return True

class Filename(param.Parameter):
    """
    Filename is a Parameter that can be set to a string specifying the
    path of a file (in unix style), and returns it in the format of
    the user's operating system.  Additionally, the specified path can
    be absolute or relative to:

    * any of the paths specified in the search_paths attribute;

    * any of the paths searched by resolve_path() (see doc for that
      function).
    """
    __slots__ = ['search_paths']

    def __init__(self,default=None,search_paths=[],**params):
        self.search_paths = search_paths
        super(Filename,self).__init__(default,**params)

    def __set__(self,obj,val):
        """
        Call Parameter's __set__, but warn if the file cannot be found.
        """
        try:
            resolve_path(val,self.search_paths)
        except IOError, e:
            param.Parameterized(name="%s.%s"%(obj.name,self._attrib_name)).warning('%s'%(e.args[0]))

        super(Filename,self).__set__(obj,val)

    def __get__(self,obj,objtype):
        """
        Return an absolute, normalized path (see resolve_path).
        """
        raw_path = super(Filename,self).__get__(obj,objtype)
        return resolve_path(raw_path,self.search_paths)

    def __getstate__(self):
        # don't want to pickle the search_paths
        state = super(Filename,self).__getstate__()
        # CBALERT: uncommenting gives an error on make tests (testsnapshots.py).
        # Testsnapshots.py runs fine on its own, and the snapshot-tests pass.
        # Must be a test interaction? Needs investigating.
        #del state['search_paths']
        return state

import topo
package_path = os.path.split(topo.__file__)[0]

# CEBALERT: we should try to remove this as soon as possible; anything
# relying on the location of the 'topographica' script should be
# re-thought (removed, or changed to output_path or package_path).
application_path = os.path.split(os.path.split(sys.executable)[0])[0]

# Location in which to create files; defaults to application_path.  If
# that's not writable, uses ~/topographica (creating it if necessary).
if is_writable(application_path):
    output_path = application_path
else:
    home_topographica = os.path.join(os.path.expanduser("~"),'topographica')
    if not os.path.exists(home_topographica):
        os.mkdir(home_topographica)
    output_path = home_topographica

def resolve_path(path,search_paths=None):
    """
    Find the path to an existing file, searching in the specified
    search paths if the filename is not absolute, and converting a
    UNIX-style path to the current OS's format if necessary.

    To turn a supplied relative path into an absolute one, the path is
    appended to each path in (search_paths+the current working
    directory+the application's base path), in that order, until the
    file is found.

    (Similar to Python's os.path.abspath(), except more search paths
    than just os.getcwd() can be used, and the file must exist.)

    An IOError is raised if the file is not found anywhere.
    """
    path = os.path.normpath(path)

    if os.path.isabs(path):
        if os.path.isfile(path):
            return path
        else:
            raise IOError('File "%s" not found.'%path)
    else:
        all_search_paths = search_paths or [] + [os.getcwd()] + [output_path] + [package_path] + [application_path]

        paths_tried = []
        for prefix in all_search_paths:
            try_path = os.path.join(os.path.normpath(prefix),path)
            if os.path.isfile(try_path): return try_path
            paths_tried.append(try_path)

        raise IOError('File "'+os.path.split(path)[1]+'" was not found in the following place(s): '+str(paths_tried)+'.')

def normalize_path(path="",prefix=None):
    """
    Convert a UNIX-style path to the current OS's format,
    typically for creating a new file or directory.

    If the path is not already absolute, it will be made
    absolute (using the specified prefix, which defaults
    to filepath.output_path) in the process.

    (Should do the same as Python's os.path.abspath(), except
    using the specified prefix rather than os.getcwd().)
    """
    if not prefix:
        prefix = output_path

    if not os.path.isabs(path):
        path = os.path.join(os.path.normpath(prefix),path)

    return os.path.normpath(path)
<|endoftext|>


<commit_before>import json
import requests
from urllib import quote_plus

class Cymon(object):

    def __init__(self, auth_token=None, endpoint='https://cymon.io/api/nexus/v1'):
        self.endpoint = endpoint
        self.session = requests.Session()
        self.session.headers = {
            'content-type': 'application/json',
            'accept': 'application/json',
        }
        if auth_token:
            self.session.headers.update({'Authorization': 'Token {0}'.format(auth_token)})

    def get(self, method, params=None):
        r = self.session.get(self.endpoint + method, params=params)
        r.raise_for_status()
        return r

    def post(self, method, params, headers=None):
        r = self.session.post(self.endpoint + method, data=json.dumps(params), headers=headers)
        r.raise_for_status()
        return r

    def list(self):
        r = self.get('')
        return json.loads(r.text)

    def ip_lookup(self, ip_addr):
        r = self.get('/ip/' + ip_addr)
        return json.loads(r.text)

    def ip_events(self, ip_addr):
        r = self.get('/ip/' + ip_addr + '/events')
        return json.loads(r.text)

    def ip_domains(self, ip_addr):
        r = self.get('/ip/' + ip_addr + '/domains')
        return json.loads(r.text)

    def ip_urls(self, ip_addr):
        r = self.get('/ip/' + ip_addr + '/urls')
        return json.loads(r.text)

    def ip_timeline(self, ip_addr):
        r = self.get('/ip/' + ip_addr + '/timeline')
        return json.loads(r.text)

    def domain_lookup(self, name):
        r = self.get('/domain/' + name)
        return json.loads(r.text)

    def domain_events(self, name):
        '''Not sure if this is functioning on Cymon.io'''
        r = self.get('/domain/' + name + '/events')
        return json.loads(r.text)

    def url_lookup(self, location):
        r = self.get('/url/' + quote_plus(location))
        return json.loads(r.text)

    def ip_blacklist(self, tag, days=1):
        ''' supported tags: malware, botnet, spam, phishing, dnsbl, blacklist '''
        r = self.get('/blacklist/ip/' + tag + '/?days=%d' %(days))
        return json.loads(r.text)

    def domain_blacklist(self, tag, days=1):
        ''' supported tags: malware, botnet, spam, phishing, dnsbl, blacklist '''
        r = self.get('/blacklist/domain/' + tag + '/?days=%d' %(days))
        return json.loads(r.text)

    def domain_dga(self, name):
        r = self.get('/dga/' + name)
        return json.loads(r.text)<commit_msg>Added recent events endpoint<commit_after>import json
import requests
from urllib import quote_plus

class Cymon(object):

    def __init__(self, auth_token=None, endpoint='https://cymon.io/api/nexus/v1'):
        self.endpoint = endpoint
        self.session = requests.Session()
        self.session.headers = {
            'content-type': 'application/json',
            'accept': 'application/json',
        }
        if auth_token:
            self.session.headers.update({'Authorization': 'Token {0}'.format(auth_token)})

    def get(self, method, params=None):
        r = self.session.get(self.endpoint + method, params=params)
        r.raise_for_status()
        return r

    def post(self, method, params, headers=None):
        r = self.session.post(self.endpoint + method, data=json.dumps(params), headers=headers)
        r.raise_for_status()
        return r

    def list(self):
        r = self.get('')
        return json.loads(r.text)

    def ip_lookup(self, ip_addr):
        r = self.get('/ip/' + ip_addr)
        return json.loads(r.text)

    def ip_events(self, ip_addr):
        r = self.get('/ip/' + ip_addr + '/events')
        return json.loads(r.text)

    def ip_domains(self, ip_addr):
        r = self.get('/ip/' + ip_addr + '/domains')
        return json.loads(r.text)

    def ip_urls(self, ip_addr):
        r = self.get('/ip/' + ip_addr + '/urls')
        return json.loads(r.text)

    def ip_timeline(self, ip_addr):
        r = self.get('/ip/' + ip_addr + '/timeline')
        return json.loads(r.text)

    def domain_lookup(self, name):
        r = self.get('/domain/' + name)
        return json.loads(r.text)

    def domain_events(self, name):
        '''Not sure if this is functioning on Cymon.io'''
        r = self.get('/domain/' + name + '/events')
        return json.loads(r.text)

    def url_lookup(self, location):
        r = self.get('/url/' + quote_plus(location))
        return json.loads(r.text)

    def ip_blacklist(self, tag, days=1):
        ''' supported tags: malware, botnet, spam, phishing, dnsbl, blacklist '''
        r = self.get('/blacklist/ip/' + tag + '/?days=%d' %(days))
        return json.loads(r.text)

    def domain_blacklist(self, tag, days=1):
        ''' supported tags: malware, botnet, spam, phishing, dnsbl, blacklist '''
        r = self.get('/blacklist/domain/' + tag + '/?days=%d' %(days))
        return json.loads(r.text)

    def domain_dga(self, name):
        r = self.get('/dga/' + name)
        return json.loads(r.text)

    def recent(self):
        '''Return recently reported events as on https://cymon.io'''

        # we need to contact a different endpoint for this; saving the old for future use
        old_endpoint = self.endpoint
        self.endpoint = 'https://cymon.io/api/public/'
        r = self.get('dashboard/recent-objects/')

        # after getting a response we need to reset the endpoint to the original
        self.endpoint = old_endpoint
        return json.loads(r.text)
        <|endoftext|>


<commit_before>import struct
import sys
from cymysql.constants import ER

PYTHON3 = sys.version_info[0] > 2

if PYTHON3:
    StandardError = Exception

class MySQLError(StandardError):

    """Exception related to operation with MySQL."""
    def __init__(self, *args):
        if len(args) == 2:
            self.errno = args[0]
            self.errmsg = args[1]
        else:
            self.errno = -1
            self.errmsg = args[0]
        super(MySQLError, self).__init__(*args)

class Warning(MySQLError):

    """Exception raised for important warnings like data truncations
    while inserting, etc."""

class Error(MySQLError):

    """Exception that is the base class of all other error exceptions
    (not Warning)."""

class InterfaceError(Error):

    """Exception raised for errors that are related to the database
    interface rather than the database itself."""

class DatabaseError(Error):

    """Exception raised for errors that are related to the
    database."""

class DataError(DatabaseError):

    """Exception raised for errors that are due to problems with the
    processed data like division by zero, numeric value out of range,
    etc."""

class OperationalError(DatabaseError):

    """Exception raised for errors that are related to the database's
    operation and not necessarily under the control of the programmer,
    e.g. an unexpected disconnect occurs, the data source name is not
    found, a transaction could not be processed, a memory allocation
    error occurred during processing, etc."""

class IntegrityError(DatabaseError):

    """Exception raised when the relational integrity of the database
    is affected, e.g. a foreign key check fails, duplicate key,
    etc."""

class InternalError(DatabaseError):

    """Exception raised when the database encounters an internal
    error, e.g. the cursor is not valid anymore, the transaction is
    out of sync, etc."""

class ProgrammingError(DatabaseError):

    """Exception raised for programming errors, e.g. table not found
    or already exists, syntax error in the SQL statement, wrong number
    of parameters specified, etc."""

class NotSupportedError(DatabaseError):

    """Exception raised in case a method or database API was used
    which is not supported by the database, e.g. requesting a
    .rollback() on a connection that does not support transaction or
    has transactions turned off."""

error_map = {}

def _map_error(exc, *errors):
    for error in errors:
        error_map[error] = exc

_map_error(ProgrammingError, ER.DB_CREATE_EXISTS, ER.SYNTAX_ERROR,
           ER.PARSE_ERROR, ER.NO_SUCH_TABLE, ER.WRONG_DB_NAME,
           ER.WRONG_TABLE_NAME, ER.FIELD_SPECIFIED_TWICE,
           ER.INVALID_GROUP_FUNC_USE, ER.UNSUPPORTED_EXTENSION,
           ER.TABLE_MUST_HAVE_COLUMNS, ER.CANT_DO_THIS_DURING_AN_TRANSACTION)
_map_error(DataError, ER.WARN_DATA_TRUNCATED, ER.WARN_NULL_TO_NOTNULL,
           ER.WARN_DATA_OUT_OF_RANGE, ER.NO_DEFAULT, ER.PRIMARY_CANT_HAVE_NULL,
           ER.DATA_TOO_LONG, ER.DATETIME_FUNCTION_OVERFLOW)
_map_error(IntegrityError, ER.DUP_ENTRY, ER.NO_REFERENCED_ROW,
           ER.NO_REFERENCED_ROW_2, ER.ROW_IS_REFERENCED, ER.ROW_IS_REFERENCED_2,
           ER.CANNOT_ADD_FOREIGN, ER.BAD_NULL_ERROR, ER.DATA_OUT_OF_RANGE,
           ER.CHECK_CONSTRAINT, ER.CONSTRAINT_FAILED)
_map_error(NotSupportedError, ER.WARNING_NOT_COMPLETE_ROLLBACK,
           ER.NOT_SUPPORTED_YET, ER.FEATURE_DISABLED, ER.UNKNOWN_STORAGE_ENGINE)
_map_error(OperationalError, ER.DBACCESS_DENIED_ERROR, ER.ACCESS_DENIED_ERROR,
           ER.TABLEACCESS_DENIED_ERROR, ER.COLUMNACCESS_DENIED_ERROR,
           ER.LOCK_DEADLOCK)

del _map_error, ER

def _get_error_info(data):
    errno = struct.unpack('<h', data[1:3])[0]
    if sys.version_info[0] == 3:
        is_41 = data[3] == ord("#")
    else:
        is_41 = data[3] == "#"
    if is_41:
        # version 4.1
        sqlstate = data[4:9].decode("utf8")
        errorvalue = data[9:].decode("utf8")
        return (errno, sqlstate, errorvalue)
    else:
        # version 4.0
        return (errno, None, data[3:].decode("utf8"))

def _check_mysql_exception(errinfo):
    errno, sqlstate, errorvalue = errinfo
    errorclass = error_map.get(errno, None)
    if errorclass:
        raise errorclass(errno, errorvalue)

    # couldn't find the right error number
    raise InternalError(errno, errorvalue)

def raise_mysql_exception(data):
    errinfo = _get_error_info(data)
    _check_mysql_exception(errinfo)
<commit_msg>check constraint raise OperationalError<commit_after>import struct
import sys
from cymysql.constants import ER

PYTHON3 = sys.version_info[0] > 2

if PYTHON3:
    StandardError = Exception

class MySQLError(StandardError):

    """Exception related to operation with MySQL."""
    def __init__(self, *args):
        if len(args) == 2:
            self.errno = args[0]
            self.errmsg = args[1]
        else:
            self.errno = -1
            self.errmsg = args[0]
        super(MySQLError, self).__init__(*args)

class Warning(MySQLError):

    """Exception raised for important warnings like data truncations
    while inserting, etc."""

class Error(MySQLError):

    """Exception that is the base class of all other error exceptions
    (not Warning)."""

class InterfaceError(Error):

    """Exception raised for errors that are related to the database
    interface rather than the database itself."""

class DatabaseError(Error):

    """Exception raised for errors that are related to the
    database."""

class DataError(DatabaseError):

    """Exception raised for errors that are due to problems with the
    processed data like division by zero, numeric value out of range,
    etc."""

class OperationalError(DatabaseError):

    """Exception raised for errors that are related to the database's
    operation and not necessarily under the control of the programmer,
    e.g. an unexpected disconnect occurs, the data source name is not
    found, a transaction could not be processed, a memory allocation
    error occurred during processing, etc."""

class IntegrityError(DatabaseError):

    """Exception raised when the relational integrity of the database
    is affected, e.g. a foreign key check fails, duplicate key,
    etc."""

class InternalError(DatabaseError):

    """Exception raised when the database encounters an internal
    error, e.g. the cursor is not valid anymore, the transaction is
    out of sync, etc."""

class ProgrammingError(DatabaseError):

    """Exception raised for programming errors, e.g. table not found
    or already exists, syntax error in the SQL statement, wrong number
    of parameters specified, etc."""

class NotSupportedError(DatabaseError):

    """Exception raised in case a method or database API was used
    which is not supported by the database, e.g. requesting a
    .rollback() on a connection that does not support transaction or
    has transactions turned off."""

error_map = {}

def _map_error(exc, *errors):
    for error in errors:
        error_map[error] = exc

_map_error(ProgrammingError, ER.DB_CREATE_EXISTS, ER.SYNTAX_ERROR,
           ER.PARSE_ERROR, ER.NO_SUCH_TABLE, ER.WRONG_DB_NAME,
           ER.WRONG_TABLE_NAME, ER.FIELD_SPECIFIED_TWICE,
           ER.INVALID_GROUP_FUNC_USE, ER.UNSUPPORTED_EXTENSION,
           ER.TABLE_MUST_HAVE_COLUMNS, ER.CANT_DO_THIS_DURING_AN_TRANSACTION)
_map_error(DataError, ER.WARN_DATA_TRUNCATED, ER.WARN_NULL_TO_NOTNULL,
           ER.WARN_DATA_OUT_OF_RANGE, ER.NO_DEFAULT, ER.PRIMARY_CANT_HAVE_NULL,
           ER.DATA_TOO_LONG, ER.DATETIME_FUNCTION_OVERFLOW)
_map_error(IntegrityError, ER.DUP_ENTRY, ER.NO_REFERENCED_ROW,
           ER.NO_REFERENCED_ROW_2, ER.ROW_IS_REFERENCED, ER.ROW_IS_REFERENCED_2,
           ER.CANNOT_ADD_FOREIGN, ER.BAD_NULL_ERROR, ER.DATA_OUT_OF_RANGE)
_map_error(NotSupportedError, ER.WARNING_NOT_COMPLETE_ROLLBACK,
           ER.NOT_SUPPORTED_YET, ER.FEATURE_DISABLED, ER.UNKNOWN_STORAGE_ENGINE)
_map_error(OperationalError, ER.DBACCESS_DENIED_ERROR, ER.ACCESS_DENIED_ERROR,
           ER.TABLEACCESS_DENIED_ERROR, ER.COLUMNACCESS_DENIED_ERROR,
           ER.LOCK_DEADLOCK, ER.CHECK_CONSTRAINT, ER.CONSTRAINT_FAILED)

del _map_error, ER

def _get_error_info(data):
    errno = struct.unpack('<h', data[1:3])[0]
    if sys.version_info[0] == 3:
        is_41 = data[3] == ord("#")
    else:
        is_41 = data[3] == "#"
    if is_41:
        # version 4.1
        sqlstate = data[4:9].decode("utf8")
        errorvalue = data[9:].decode("utf8")
        return (errno, sqlstate, errorvalue)
    else:
        # version 4.0
        return (errno, None, data[3:].decode("utf8"))

def _check_mysql_exception(errinfo):
    errno, sqlstate, errorvalue = errinfo
    errorclass = error_map.get(errno, None)
    if errorclass:
        raise errorclass(errno, errorvalue)

    # couldn't find the right error number
    raise InternalError(errno, errorvalue)

def raise_mysql_exception(data):
    errinfo = _get_error_info(data)
    _check_mysql_exception(errinfo)
<|endoftext|>


<commit_before>#!/usr/bin/python -OO

# This file is part of Archivematica.
#
# Copyright 2010-2013 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

# @package Archivematica
# @subpackage MCPServer
# @author Joseph Perry <joseph@artefactual.com>

from unit import unit
from unitFile import unitFile
import uuid
import archivematicaMCP
import os
import time
import sys
import traceback
import pyinotify
import threading
import shutil
import MySQLdb
sys.path.append("/usr/lib/archivematica/archivematicaCommon")
import lxml.etree as etree
from fileOperations import renameAsSudo
from databaseFunctions import insertIntoEvents
from databaseFunctions import deUnicode

sys.path.append("/usr/share/archivematica/dashboard")
from main.models import Transfer

class unitTransfer(unit):
    def __init__(self, currentPath, UUID=""):
        self.owningUnit = None
        self.unitType = "Transfer"
        #Just Use the end of the directory name
        self.pathString = "%transferDirectory%"
        currentPath2 = currentPath.replace(archivematicaMCP.config.get('MCPServer', "sharedDirectory"), \
                       "%sharedPath%", 1)

        if not UUID:
            try:
                transfer = Transfer.objects.get(currentlocation=currentPath2)
                UUID = transfer.uuid
                print "Opening existing Transfer:", UUID, "-", currentPath2
            except Transfer.DoesNotExist:
                pass

        if not UUID:
            uuidLen = -36
            if archivematicaMCP.isUUID(currentPath[uuidLen-1:-1]):
                UUID = currentPath[uuidLen-1:-1]
            else:
                UUID = str(uuid.uuid4())
                self.UUID = UUID
                Transfer.objects.create(uuid=UUID, currentlocation=currentPath2)

        self.currentPath = currentPath2
        self.UUID = UUID
        self.fileList = {}

    def updateLocation(self, newLocation):
        self.currentPath = newLocation
        transfer = Transfer.objects.get(uuid=self.UUID)
        transfer.currentlocation = newLocation
        transfer.save()

    def setMagicLink(self, link, exitStatus=""):
        """Assign a link to the unit to process when loaded.
        Deprecated! Replaced with Set/Load Unit Variable"""
        transfer = Transfer.objects.get(uuid=self.UUID)
        transfer.magiclink = link
        if exitStatus:
            transfer.magiclinkexitmessage = exitStatus
        transfer.save()

    def getMagicLink(self):
        """Load a link from the unit to process.
        Deprecated! Replaced with Set/Load Unit Variable"""
        try:
            transfer = Transfer.objects.get(uuid=self.UUID)
        except Transfer.DoesNotExist:
            return
        return (transfer.magiclink, transfer.magiclinkexitmessage)

    def reload(self):
        transfer = Transfer.objects.get(uuid=self.UUID)
        self.currentPath = transfer.currentlocation

    def getReplacementDic(self, target):
        # self.currentPath = currentPath.__str__()
        # self.UUID = uuid.uuid4().__str__()
        #Pre do some variables, that other variables rely on, because dictionaries don't maintain order
        SIPUUID = self.UUID
        if self.currentPath.endswith("/"):
            SIPName = os.path.basename(self.currentPath[:-1]).replace("-" + SIPUUID, "")
        else:
            SIPName = os.path.basename(self.currentPath).replace("-" + SIPUUID, "")
        SIPDirectory = self.currentPath.replace(archivematicaMCP.config.get('MCPServer', "sharedDirectory"), "%sharedPath%")
        relativeDirectoryLocation = target.replace(archivematicaMCP.config.get('MCPServer', "sharedDirectory"), "%sharedPath%")

        ret = { \
        "%SIPLogsDirectory%": SIPDirectory + "logs/", \
        "%SIPObjectsDirectory%": SIPDirectory + "objects/", \
        "%SIPDirectory%": SIPDirectory, \
        "%transferDirectory%": SIPDirectory, \
        "%SIPDirectoryBasename%": os.path.basename(os.path.abspath(SIPDirectory)), \
        "%relativeLocation%": target.replace(self.currentPath, relativeDirectoryLocation, 1), \
        "%processingDirectory%": archivematicaMCP.config.get('MCPServer', "processingDirectory"), \
        "%checksumsNoExtention%":archivematicaMCP.config.get('MCPServer', "checksumsNoExtention"), \
        "%watchDirectoryPath%":archivematicaMCP.config.get('MCPServer', "watchDirectoryPath"), \
        "%rejectedDirectory%":archivematicaMCP.config.get('MCPServer', "rejectedDirectory"), \
        "%SIPUUID%":SIPUUID, \
        "%SIPName%":SIPName, \
        "%unitType%":self.unitType \
        }
        return ret

    def xmlify(self):
        ret = etree.Element("unit")
        etree.SubElement(ret, "type").text = "Transfer"
        unitXML = etree.SubElement(ret, "unitXML")
        etree.SubElement(unitXML, "UUID").text = self.UUID
        tempPath = self.currentPath.replace(archivematicaMCP.config.get('MCPServer', "sharedDirectory"), "%sharedPath%").decode("utf-8")
        etree.SubElement(unitXML, "currentPath").text = tempPath

        return ret

<commit_msg>unitTransfer: remove unused imports<commit_after>#!/usr/bin/python -OO

# This file is part of Archivematica.
#
# Copyright 2010-2013 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

# @package Archivematica
# @subpackage MCPServer
# @author Joseph Perry <joseph@artefactual.com>

from unit import unit
import uuid
import archivematicaMCP
import os
import sys

import lxml.etree as etree

sys.path.append("/usr/share/archivematica/dashboard")
from main.models import Transfer

class unitTransfer(unit):
    def __init__(self, currentPath, UUID=""):
        self.owningUnit = None
        self.unitType = "Transfer"
        #Just Use the end of the directory name
        self.pathString = "%transferDirectory%"
        currentPath2 = currentPath.replace(archivematicaMCP.config.get('MCPServer', "sharedDirectory"), \
                       "%sharedPath%", 1)

        if not UUID:
            try:
                transfer = Transfer.objects.get(currentlocation=currentPath2)
                UUID = transfer.uuid
                print "Opening existing Transfer:", UUID, "-", currentPath2
            except Transfer.DoesNotExist:
                pass

        if not UUID:
            uuidLen = -36
            if archivematicaMCP.isUUID(currentPath[uuidLen-1:-1]):
                UUID = currentPath[uuidLen-1:-1]
            else:
                UUID = str(uuid.uuid4())
                self.UUID = UUID
                Transfer.objects.create(uuid=UUID, currentlocation=currentPath2)

        self.currentPath = currentPath2
        self.UUID = UUID
        self.fileList = {}

    def updateLocation(self, newLocation):
        self.currentPath = newLocation
        transfer = Transfer.objects.get(uuid=self.UUID)
        transfer.currentlocation = newLocation
        transfer.save()

    def setMagicLink(self, link, exitStatus=""):
        """Assign a link to the unit to process when loaded.
        Deprecated! Replaced with Set/Load Unit Variable"""
        transfer = Transfer.objects.get(uuid=self.UUID)
        transfer.magiclink = link
        if exitStatus:
            transfer.magiclinkexitmessage = exitStatus
        transfer.save()

    def getMagicLink(self):
        """Load a link from the unit to process.
        Deprecated! Replaced with Set/Load Unit Variable"""
        try:
            transfer = Transfer.objects.get(uuid=self.UUID)
        except Transfer.DoesNotExist:
            return
        return (transfer.magiclink, transfer.magiclinkexitmessage)

    def reload(self):
        transfer = Transfer.objects.get(uuid=self.UUID)
        self.currentPath = transfer.currentlocation

    def getReplacementDic(self, target):
        # self.currentPath = currentPath.__str__()
        # self.UUID = uuid.uuid4().__str__()
        #Pre do some variables, that other variables rely on, because dictionaries don't maintain order
        SIPUUID = self.UUID
        if self.currentPath.endswith("/"):
            SIPName = os.path.basename(self.currentPath[:-1]).replace("-" + SIPUUID, "")
        else:
            SIPName = os.path.basename(self.currentPath).replace("-" + SIPUUID, "")
        SIPDirectory = self.currentPath.replace(archivematicaMCP.config.get('MCPServer', "sharedDirectory"), "%sharedPath%")
        relativeDirectoryLocation = target.replace(archivematicaMCP.config.get('MCPServer', "sharedDirectory"), "%sharedPath%")

        ret = { \
        "%SIPLogsDirectory%": SIPDirectory + "logs/", \
        "%SIPObjectsDirectory%": SIPDirectory + "objects/", \
        "%SIPDirectory%": SIPDirectory, \
        "%transferDirectory%": SIPDirectory, \
        "%SIPDirectoryBasename%": os.path.basename(os.path.abspath(SIPDirectory)), \
        "%relativeLocation%": target.replace(self.currentPath, relativeDirectoryLocation, 1), \
        "%processingDirectory%": archivematicaMCP.config.get('MCPServer', "processingDirectory"), \
        "%checksumsNoExtention%":archivematicaMCP.config.get('MCPServer', "checksumsNoExtention"), \
        "%watchDirectoryPath%":archivematicaMCP.config.get('MCPServer', "watchDirectoryPath"), \
        "%rejectedDirectory%":archivematicaMCP.config.get('MCPServer', "rejectedDirectory"), \
        "%SIPUUID%":SIPUUID, \
        "%SIPName%":SIPName, \
        "%unitType%":self.unitType \
        }
        return ret

    def xmlify(self):
        ret = etree.Element("unit")
        etree.SubElement(ret, "type").text = "Transfer"
        unitXML = etree.SubElement(ret, "unitXML")
        etree.SubElement(unitXML, "UUID").text = self.UUID
        tempPath = self.currentPath.replace(archivematicaMCP.config.get('MCPServer', "sharedDirectory"), "%sharedPath%").decode("utf-8")
        etree.SubElement(unitXML, "currentPath").text = tempPath

        return ret

<|endoftext|>


<commit_before>from xos.settings import STATISTICS_DRIVER

if (STATISTICS_DRIVER=="ceilometer"):
    from observer import ceilometer
    driver = ceilometer.CeilometerDriver()
elif (not STATISTICS_DRIVER) or (STATISTICS_DRIVER.lower() == "none"):
    # disabled
    driver = None
else:
    driver = None
    print "WARNING: Unknown statistics driver %s" % STATISTICS_DRIVER
<commit_msg>Delete __init__.py<commit_after><|endoftext|>


<commit_before>__author__ = "nikolojedison"
import math

import wpilib
from wpilib.command import Subsystem

from utilities.drive_control import *
from commands.manual.octo_drive_with_joystick import OctoDriveWithJoystick
from utilities.imu_simple import IMUSimple

class GyroDummy:
    """Makes the sim happy. Written by Aux, copied from 2015 code"""
    n = 0
    def getYaw(self):
        n = self.n
        n = n+1
        if n > 180:
            n = n-360
        elif n < -180:
            m = n+360
        self.n = n
        return n

class Drivetrain(Subsystem):
    '''Class drivetrain uses many things to do many other things.'''

    def __init__(self, robot):
        super().__init__()
        self.robot = robot

      #  if robot.isReal():
       #     self.gyro = IMUSimple()
       # else:
       #     self.gyro = GyroDummy()

        self.gyro = GyroDummy()

        #Gyro definitions, I think...
        self.x = 0
        self.y = 0
        self.rotation = 0

        #Motor definitions
        self.zed = wpilib.CANTalon(0)
        self.one = wpilib.CANTalon(1)
        self.two = wpilib.CANTalon(2)
        self.three = wpilib.CANTalon(3)
        self.four = wpilib.CANTalon(4)
        self.five = wpilib.CANTalon(5)
        self.six = wpilib.CANTalon(6)
        self.seven = wpilib.CANTalon(7)

        self.drive_diagonal = wpilib.RobotDrive(self.zed, self.two, self.six, self.one)
        self.drive_x = wpilib.RobotDrive(self.three, self.five)
        self.drive_y = wpilib.RobotDrive(self.four, self.seven)

        self.drive_diagonal.setInvertedMotor(self.drive_diagonal.MotorType.kRearLeft, True)
        self.drive_diagonal.setInvertedMotor(self.drive_diagonal.MotorType.kRearRight, True)

    def initDefaultCommand(self):
        '''When no other command is running, let the operator drive around using the joystick.'''
        self.setDefaultCommand(OctoDriveWithJoystick(self.robot))

    def log(self):
        pass
        #Still no logging implementation.

    def driveJoystick(self, joystick):
        #Start precision mode
	    precision = True

        #set the axes & pass them to the drive_control utility library
        #-----------------------------------------------------------------------
        #the getX() and getY() bits are both 0 indexed according to the joystick
        #documentation for WPIlib, so X = 0, Y = 1, Z = 2, twist (supposedly) =
        #3, etc. The thing is, the default throttle axis is set to 3, and the
        #default twist axis is set to 2.
        #So.
        #When we tested previous versions on the robot with getRawAxis(3), the
        #throttle would do the things (different joystick setup on Lopez Jr
        #iirc, so that's part of it).
        #Now (1/5/16), I've set getRawAxis(2). Works fine.

	    x = drive_control(joystick.getX()*2, precision)
	    y = drive_control(joystick.getY()*2, precision)
	    z = precision_mode(dead_zone(joystick.getRawAxis(2)*2, .1), precision)
	    #a = self.gyro.getYaw()
	    a = 0
	    self.driveManual(x, y, z, a)
	    if x>1:
		    x=1
	    elif x<-1:
		    x=-1
    #def driveManual(self, x, y, z):
    def driveManual(self, x, y, rotation, a):
        self.x, self.y, self.rotation, self.a = x, y, rotation, a
        self.drive_diagonal.mecanumDrive_Cartesian(x, y, rotation, a)
        self.drive_x.arcadeDrive(y, -rotation)
        self.drive_y.arcadeDrive(x, rotation)

            #self.x, self.y, self.z = x, y, z
            #self.four.set(x*.8)
            #self.seven.set(-x*.8)
            #self.five.set(-y*.8)
            #self.three.set(y*.8)

            #What happens here works fine. Don't question it.

            #if x > 0.0625 or x < -0.0625 :
            #    self.six.set(-x)
            #    self.one.set(-x)
            #    self.four.set(x*.8)
            #    self.seven.set(-x*.8)
            #    self.two.set(x)
            #    self.zed.set(x)

        #    elif y > 0.0625 or y < -0.0625:
            #    self.six.set(-y)
            #    self.one.set(y)
            #    self.five.set(-y*.8)
            #    self.three.set(y*.8)
            #    self.two.set(y)
            #    self.zed.set(-y)

        #    elif z > 0.0625 or z < -0.0625:
            #    self.zed.set(z)
            #    self.one.set(z)
            #    self.two.set(z)
            #    self.three.set(-z)
            #    self.four.set(z)
            #    self.six.set(z)
            #    self.five.set(-z)
            #    self.seven.set(z)

            #if all the joystick inputs aren't doing the things, set all the
            #motors to 0
        #    else:
            #    self.zed.set(0)
            #    self.one.set(0)
            #    self.two.set(0)
            #    self.three.set(0)
            #    self.four.set(0)
            #    self.five.set(0)
            #    self.six.set(0)
            #    self.seven.set(0)
<commit_msg>inversion again<commit_after>__author__ = "nikolojedison"
import math

import wpilib
from wpilib.command import Subsystem

from utilities.drive_control import *
from commands.manual.octo_drive_with_joystick import OctoDriveWithJoystick
from utilities.imu_simple import IMUSimple

class GyroDummy:
    """Makes the sim happy. Written by Aux, copied from 2015 code"""
    n = 0
    def getYaw(self):
        n = self.n
        n = n+1
        if n > 180:
            n = n-360
        elif n < -180:
            m = n+360
        self.n = n
        return n

class Drivetrain(Subsystem):
    '''Class drivetrain uses many things to do many other things.'''

    def __init__(self, robot):
        super().__init__()
        self.robot = robot

      #  if robot.isReal():
       #     self.gyro = IMUSimple()
       # else:
       #     self.gyro = GyroDummy()

        self.gyro = GyroDummy()

        #Gyro definitions, I think...
        self.x = 0
        self.y = 0
        self.rotation = 0

        #Motor definitions
        self.zed = wpilib.CANTalon(0)
        self.one = wpilib.CANTalon(1)
        self.two = wpilib.CANTalon(2)
        self.three = wpilib.CANTalon(3)
        self.four = wpilib.CANTalon(4)
        self.five = wpilib.CANTalon(5)
        self.six = wpilib.CANTalon(6)
        self.seven = wpilib.CANTalon(7)

        self.drive_diagonal = wpilib.RobotDrive(self.zed, self.two, self.six, self.one)
        self.drive_x = wpilib.RobotDrive(self.three, self.five)
        self.drive_y = wpilib.RobotDrive(self.four, self.seven)

        self.drive_diagonal.setInvertedMotor(self.drive_diagonal.MotorType.kRearLeft, True)
        self.drive_diagonal.setInvertedMotor(self.drive_diagonal.MotorType.kRearRight, True)

    def initDefaultCommand(self):
        '''When no other command is running, let the operator drive around using the joystick.'''
        self.setDefaultCommand(OctoDriveWithJoystick(self.robot))

    def log(self):
        pass
        #Still no logging implementation.

    def driveJoystick(self, joystick):
        #Start precision mode
	    precision = True

        #set the axes & pass them to the drive_control utility library
        #-----------------------------------------------------------------------
        #the getX() and getY() bits are both 0 indexed according to the joystick
        #documentation for WPIlib, so X = 0, Y = 1, Z = 2, twist (supposedly) =
        #3, etc. The thing is, the default throttle axis is set to 3, and the
        #default twist axis is set to 2.
        #So.
        #When we tested previous versions on the robot with getRawAxis(3), the
        #throttle would do the things (different joystick setup on Lopez Jr
        #iirc, so that's part of it).
        #Now (1/5/16), I've set getRawAxis(2). Works fine.

	    x = drive_control(joystick.getX()*2, precision)
	    y = drive_control(joystick.getY()*2, precision)
	    z = precision_mode(dead_zone(joystick.getRawAxis(2)*2, .1), precision)
	    #a = self.gyro.getYaw()
	    a = 0
	    self.driveManual(x, y, z, a)
	    if x>1:
		    x=1
	    elif x<-1:
		    x=-1

    #invert zed and one for rotation
    #def driveManual(self, x, y, z):
    def driveManual(self, x, y, rotation, a):
        self.x, self.y, self.rotation, self.a = x, y, rotation, a
        self.drive_diagonal.mecanumDrive_Cartesian(x, y, rotation, a)
        self.drive_x.arcadeDrive(y, -rotation)
        self.drive_y.arcadeDrive(x, rotation)
        if rotation < 0.0625 or > 0.0625:
            self.drive_diagonal.setInvertedMotor(self.drive_diagonal.MotorType.kFrontLeft, True)
            self.drive_diagonal.setInvertedMotor(self.drive_diagonal.MotorType.kFrontRight, True)
        else:
            self.drive_diagonal.setInvertedMotor(self.drive_diagonal.MotorType.kFrontLeft, False)
            self.drive_diagonal.setInvertedMotor(self.drive_diagonal.MotorType.kFrontRight, False)

            #self.x, self.y, self.z = x, y, z
            #self.four.set(x*.8)
            #self.seven.set(-x*.8)
            #self.five.set(-y*.8)
            #self.three.set(y*.8)

            #What happens here works fine. Don't question it.

            #if x > 0.0625 or x < -0.0625 :
            #    self.six.set(-x)
            #    self.one.set(-x)
            #    self.four.set(x*.8)
            #    self.seven.set(-x*.8)
            #    self.two.set(x)
            #    self.zed.set(x)

        #    elif y > 0.0625 or y < -0.0625:
            #    self.six.set(-y)
            #    self.one.set(y)
            #    self.five.set(-y*.8)
            #    self.three.set(y*.8)
            #    self.two.set(y)
            #    self.zed.set(-y)

        #    elif z > 0.0625 or z < -0.0625:
            #    self.zed.set(z)
            #    self.one.set(z)
            #    self.two.set(z)
            #    self.three.set(-z)
            #    self.four.set(z)
            #    self.six.set(z)
            #    self.five.set(-z)
            #    self.seven.set(z)

            #if all the joystick inputs aren't doing the things, set all the
            #motors to 0
        #    else:
            #    self.zed.set(0)
            #    self.one.set(0)
            #    self.two.set(0)
            #    self.three.set(0)
            #    self.four.set(0)
            #    self.five.set(0)
            #    self.six.set(0)
            #    self.seven.set(0)
<|endoftext|>


<commit_before>__author__ = 'marcusmorgenstern'
__mail__ = ''

import ROOT
from PyAnalysisTools.base import _logger, InvalidInputError
from PyAnalysisTools.PlottingUtils.PlotConfig import parse_and_build_plot_config
from PyAnalysisTools.ROOTUtils.FileHandle import FileHandle
from PyAnalysisTools.PlottingUtils import Formatting as FM
from PyAnalysisTools.PlottingUtils import HistTools as HT

class BasePlotter(object):
    def __init__(self, **kwargs):
        if not "input_file" in kwargs:
            _logger.error("No input files provided")
            raise  InvalidInputError("No input files")
        if not "plot_config_file" in kwargs:
            _logger.error("No plot config file provided. Nothing to parse")
            raise InvalidInputError("No plot config file")
        for k,v in kwargs.iteritems():
            setattr(self, k, v)
        self.file_handle = FileHandle(self.input_file)

    def parse_plot_config(self):
        _logger.debug("Try to parse plot config file")
        self.plot_configs = parse_and_build_plot_config(self.plot_config_file)

    def initialise(self):
        self.parse_plot_config()

    @staticmethod
    def get_histogram_definition(plot_config):
        dimension = plot_config.dist.count(":")
        hist = None
        if dimension == 0:
            hist = ROOT.TH1F(plot_config.name, "", plot_config.bins, plot_config.xmin, plot_config.xmax)
        elif dimension == 1:
            hist = ROOT.TH2F(plot_config.name, "", plot_config.bins, plot_config.xmin, plot_config.xmax,
                             plot_config.ybins, plot_config.ymin, plot_config.ymax)
        elif dimension==2:
            hist = ROOT.TH3F(plot_config.name, "", plot_config.bins, plot_config.xmin, plot_config.xmax,
                             plot_config.ybins, plot_config.ymin, plot_config.ymax,
                             plot_config.zbins, plot_config.zmin, plot_config.zmax)
        if not hist:
            _logger.error("Unable to create histogram for plot_config %s for variable %s" % (plot_config.name,
                                                                                plot_config.dist))
            raise InvalidInputError("Invalid plot configuration")
        return hist

    def make_plot(self, plot_config):
        hist = self.__class__.get_histogram_definition(plot_config)
        print hist
        try:
            self.file_handle.fetch_and_link_hist_to_tree(self.tree_name, hist, plot_config.dist, plot_config.cuts)
        except Exception as e:
            raise e
        self.format_hist(hist, plot_config)
        hist.Draw()
        raw_input()

    def format_hist(self, hist, plot_config):
        if hasattr(plot_config, "xtitle"):
            if hasattr(plot_config, "unit"):
                plot_config.xtitle += " [" + plot_config.unit + "]"
            FM.set_title_x(hist, plot_config.xtitle)
        y_title = "Entries"
        if hasattr(plot_config, "ytitle"):
            y_title = plot_config.ytitle
        if hasattr(plot_config, "unit"):
            y_title += " / %.1f %s" % (hist.GetXaxis().GetBinWidth(0), plot_config.unit)
        FM.set_title_y(hist, y_title)
        if hasattr(plot_config, "rebin"):
            HT.rebin(hist, plot_config.rebin)

    def make_plots(self):
        for plot_config in self.plot_configs:
            self.make_plot(plot_config)
<commit_msg>implement functionality for multiple inputs<commit_after>__author__ = 'marcusmorgenstern'
__mail__ = ''

import ROOT
from PyAnalysisTools.base import _logger, InvalidInputError
from PyAnalysisTools.PlottingUtils.PlotConfig import parse_and_build_plot_config, parse_and_build_process_config
from PyAnalysisTools.ROOTUtils.FileHandle import FileHandle
from PyAnalysisTools.PlottingUtils import Formatting as FM
from PyAnalysisTools.PlottingUtils import HistTools as HT
from PyAnalysisTools.PlottingUtils import PlottingTools as PT
from PyAnalysisTools.AnalysisTools.XSHandle import XSHandle

class BasePlotter(object):
    def __init__(self, **kwargs):
        if "input_files" not in kwargs:
            _logger.error("No input files provided")
            raise  InvalidInputError("No input files")
        if "plot_config_file" not in kwargs:
            _logger.error("No plot config file provided. Nothing to parse")
            raise InvalidInputError("No plot config file")
        if "process_config_file" not in kwargs:
            _logger.warning("No process config file provided. Unable to read process specific options.")
        kwargs.setdefault("process_config_file", None)
        kwargs.setdefault("xs_config_file", None)
        kwargs.setdefault("batch", False)
        for k,v in kwargs.iteritems():
            setattr(self, k, v)
        ROOT.gROOT.SetBatch(self.batch)
        self.file_handles = [FileHandle(input_file) for input_file in self.input_files]
        self.xs_handle = XSHandle()
        self.process_config = self.parse_process_config()
        FM.load_atlas_style()
        self.histograms = {}

    def parse_plot_config(self):
        _logger.debug("Try to parse plot config file")
        self.plot_configs, self.common_config = parse_and_build_plot_config(self.plot_config_file)

    def parse_process_config(self):
        if self.process_config_file is None:
            return None
        process_config = parse_and_build_process_config(self.process_config_file, self.xs_config_file)
        return process_config

    def initialise(self):
        self.parse_plot_config()

    @staticmethod
    def get_histogram_definition(plot_config):
        dimension = plot_config.dist.count(":")
        hist = None
        hist_name = plot_config.name
        if dimension == 0:
            hist = ROOT.TH1F(hist_name, "", plot_config.bins, plot_config.xmin, plot_config.xmax)
        elif dimension == 1:
            hist = ROOT.TH2F(hist_name, "", plot_config.bins, plot_config.xmin, plot_config.xmax,
                             plot_config.ybins, plot_config.ymin, plot_config.ymax)
        elif dimension==2:
            hist = ROOT.TH3F(hist_name, "", plot_config.bins, plot_config.xmin, plot_config.xmax,
                             plot_config.ybins, plot_config.ymin, plot_config.ymax,
                             plot_config.zbins, plot_config.zmin, plot_config.zmax)
        if not hist:
            _logger.error("Unable to create histogram for plot_config %s for variable %s" % (plot_config.name,
                                                                                plot_config.dist))
            raise InvalidInputError("Invalid plot configuration")
        return hist

    def retrieve_histogram(self, file_handle, plot_config):
        hist = self.__class__.get_histogram_definition(plot_config)
        try:
            file_handle.fetch_and_link_hist_to_tree(self.tree_name, hist, plot_config.dist, plot_config.cuts)
            hist.SetName(hist.GetName() + "_" + file_handle.process)
        except Exception as e:
            raise e
        self.format_hist(hist, plot_config)
        return hist

    #todo: should be moved to formatter
    def format_hist(self, hist, plot_config):
        if hasattr(plot_config, "xtitle"):
            xtitle = plot_config.xtitle
            if hasattr(plot_config, "unit"):
                xtitle += " [" + plot_config.unit + "]"
            FM.set_title_x(hist, xtitle)
        y_title = "Entries"
        if hasattr(plot_config, "ytitle"):
            y_title = plot_config.ytitle
        if hasattr(plot_config, "unit"):
            y_title += " / %.1f %s" % (hist.GetXaxis().GetBinWidth(0), plot_config.unit)
        FM.set_title_y(hist, y_title)
        if hasattr(plot_config, "rebin"):
            HT.rebin(hist, plot_config.rebin)

    def make_plots(self):
        for file_handle in self.file_handles:
            for plot_config in self.plot_configs:
                try:
                    self.histograms[plot_config][file_handle.process] = self.retrieve_histogram(file_handle, plot_config)
                except KeyError:
                    self.histograms[plot_config] = {file_handle.process: self.retrieve_histogram(file_handle, plot_config)}

        for plot_config_name, data in self.histograms.iteritems():
            canvas = PT.plot_histograms(data, plot_config, self.common_config, self.process_config)
            canvas.Update()
            raw_input()
<|endoftext|>


<commit_before>'''The API calls'''

import StringIO
from flask import (
    jsonify,
    request,
    send_file,
)
import sqlalchemy
from . import app
from .helpers import get_db
from .search import (
    search_bgcs,
    create_cluster_csv,
    available_term_by_category,
)
import sql

@app.route('/api/v1.0/version')
def get_version():
    '''display the API version'''
    from .version import __version__ as api_version
    ret = {
        'api': api_version,
        'sqlalchemy': sqlalchemy.__version__
    }
    return jsonify(ret)

@app.route('/api/v1.0/stats')
def get_stats():
    '''contents for the stats page'''
    cur = get_db().cursor()
    cur.execute(sql.STATS_CLUSTER_COUNT)
    num_clusters = cur.fetchone()[0]

    cur.execute(sql.STATS_GENOME_COUNT)
    num_genomes = cur.fetchone()[0]

    cur.execute(sql.STATS_SEQUENCE_COUNT)
    num_sequences = cur.fetchone()[0]

    clusters = []

    cur.execute(sql.STATS_COUNTS_BY_TYPE)
    ret = cur.fetchall()
    for cluster in ret:
        clusters.append({'name': cluster.term, 'description': cluster.description, 'count': cluster.count})

    cur.execute(sql.STATS_TAXON_SEQUENCES)
    ret = cur.fetchone()
    top_seq_taxon = ret.tax_id
    top_seq_taxon_count = ret.tax_count

    cur.execute(sql.STATS_TAXON_SECMETS)
    ret = cur.fetchone()
    top_secmet_taxon = ret.tax_id
    top_secmet_species = ret.species
    top_secmet_taxon_count = ret.clusters_per_seq

    stats = {
        'num_clusters': num_clusters,
        'num_genomes': num_genomes,
        'num_sequences': num_sequences,
        'top_seq_taxon': top_seq_taxon,
        'top_seq_taxon_count': top_seq_taxon_count,
        'top_secmet_taxon': top_secmet_taxon,
        'top_secmet_taxon_count': top_secmet_taxon_count,
        'top_secmet_species': top_secmet_species,
        'clusters': clusters,
    }

    return jsonify(stats)

@app.route('/api/v1.0/tree/secmet')
def get_sec_met_tree():
    '''Get the jsTree structure for secondary metabolite clusters'''
    cur = get_db().cursor()
    cur.execute(sql.SECMET_TREE)
    ret = cur.fetchall()

    clusters = []
    types = {}

    for entry in ret:
        types[entry.term] = entry.description
        clusters.append({
            "id": "{}_c{}_{}".format(entry.acc, entry.cluster_number, entry.term),
            "parent": entry.term,
            "text": "{} {} Cluster {}".format(entry.species, entry.acc, entry.cluster_number),
            "type": "cluster",
        })

    for name, desc in types.iteritems():
        clusters.insert(0, {
            "id": name,
            "parent": "#",
            "text": desc,
            "state": {
                "disabled": True
            }
        })

    tree = clusters
    return jsonify(tree)

@app.route('/api/v1.0/tree/taxa')
def get_taxon_tree():
    '''Get the jsTree structure for all taxa'''
    tree_id = request.args.get('id', '1')

    cur = get_db().cursor()

    HANDLERS = {
        'superkingdom': get_phylum,
        'phylum': get_class,
        'class': get_order,
        'order': get_family,
        'family': get_genus,
        'genus': get_species,
    }

    if tree_id == '1':
        tree = get_superkingdom(cur)
    else:
        params = tree_id.split('_')
        taxlevel = params[0]
        params = params[1:]
        handler = HANDLERS.get(taxlevel, lambda x, y: [])
        tree = handler(cur, params)

    return jsonify(tree)

def get_superkingdom(cur):
    '''Get list of superkingdoms'''
    tree = []
    cur.execute(sql.TAXTREE_SUPERKINGOM)
    kingdoms = cur.fetchall()
    for kingdom in kingdoms:
        tree.append(_create_tree_node('superkingdom_{}'.format(kingdom[0].lower()),
                                      '#', kingdom[0]))

    return tree

def get_phylum(cur, params):
    '''Get list of phyla per kingdom'''
    tree = []
    cur.execute(sql.TAXTREE_PHYLUM, params)
    phyla = cur.fetchall()
    for phylum in phyla:
        id_list = params + [phylum[0].lower()]
        tree.append(_create_tree_node('phylum_{}'.format('_'.join(id_list)),
                                      'superkingdom_{}'.format('_'.join(params)),
                                      phylum[0]))

    return tree

def get_class(cur, params):
    '''Get list of classes per kingdom/phylum'''
    tree = []
    cur.execute(sql.TAXTREE_CLASS, params)
    classes = cur.fetchall()
    for cls in classes:
        id_list = params + [cls[0].lower()]
        tree.append(_create_tree_node('class_{}'.format('_'.join(id_list)),
                                      'phylum_{}'.format('_'.join(params)),
                                      cls[0]))
    return tree

def get_order(cur, params):
    '''Get list of oders per kingdom/phylum/class'''
    tree = []
    cur.execute(sql.TAXTREE_ORDER, params)
    orders = cur.fetchall()
    for order in orders:
        id_list = params + [order[0].lower()]
        tree.append(_create_tree_node('order_{}'.format('_'.join(id_list)),
                                      'class_{}'.format('_'.join(params)),
                                      order[0]))
    return tree

def get_family(cur, params):
    '''Get list of families per kingdom/phylum/class/order'''
    tree = []
    cur.execute(sql.TAXTREE_FAMILY, params)
    families = cur.fetchall()
    for family in families:
        id_list = params + [family[0].lower()]
        tree.append(_create_tree_node('family_{}'.format('_'.join(id_list)),
                                      'order_{}'.format('_'.join(params)),
                                      family[0]))
    return tree

def get_genus(cur, params):
    '''Get list of genera per kingdom/phylum/class/order/family'''
    tree = []
    cur.execute(sql.TAXTREE_GENUS, params)
    genera = cur.fetchall()
    for genus in genera:
        id_list = params + [genus[0].lower()]
        tree.append(_create_tree_node('genus_{}'.format('_'.join(id_list)),
                                      'family_{}'.format('_'.join(params)),
                                      genus[0]))
    return tree

def get_species(cur, params):
    '''Get list of species per kingdom/phylum/class/order/family/genus'''
    tree = []
    cur.execute(sql.TAXTREE_SPECIES, params)
    strains = cur.fetchall()
    for strain in strains:
        tree.append(_create_tree_node('{}'.format(strain.acc.lower()),
                                      'genus_{}'.format('_'.join(params)),
                                      '{} {}.{}'.format(strain.species, strain.acc, strain.version),
                                      disabled=False, leaf=True))

    return tree

@app.route('/api/v1.0/search', methods=['POST'])
def search():
    '''Handle searching the database'''
    search_string = request.json.get('search_string', '')
    try:
        offset = int(request.json.get('offset'))
    except TypeError:
        offset = 0

    try:
        paginate = int(request.json.get('paginate'))
    except TypeError:
        paginate = 50

    total_count, stats, found_bgcs = search_bgcs(search_string, offset=offset, paginate=paginate)

    result = {
        'total': total_count,
        'clusters': found_bgcs,
        'offset': offset,
        'paginate': paginate,
        'stats': stats,
    }

    return jsonify(result)

@app.route('/api/v1.0/export', methods=['POST'])
def export():
    '''Export the search results as CSV file'''
    search_string = request.json.get('search_string', '')
    _, _, found_bgcs = search_bgcs(search_string, mapfunc=create_cluster_csv)

    found_bgcs.insert(0, '#Species\tNCBI accession\tCluster number\tBGC type\tFrom\tTo\tMost similar known cluster\tSimilarity in %\tMIBiG BGC-ID\tResults URL')

    handle = StringIO.StringIO()
    for line in found_bgcs:
        handle.write('{}\n'.format(line))

    handle.seek(0)
    return send_file(handle, attachment_filename='asdb_search_results.csv', as_attachment=True)

@app.route('/api/v1.0/genome/<identifier>')
def show_genome(identifier):
    '''show information for a genome by identifier'''
    search_string = '[acc]{}'.format(identifier)
    _, _, found_bgcs = search_bgcs(search_string)

    return jsonify(found_bgcs)

@app.route('/api/v1.0/available/<category>/<term>')
def list_available(category, term):
    '''list available terms for a given category'''
    return jsonify(available_term_by_category(category, term))

def _create_tree_node(node_id, parent, text, disabled=True, leaf=False):
    '''create a jsTree node structure'''
    ret = {}
    ret['id'] = node_id
    ret['parent'] = parent
    ret['text'] = text
    if disabled:
        ret['state'] = {'disabled': True}
    if leaf:
        ret['type'] = 'strain'
    else:
        ret['children'] = True
    return ret
<commit_msg>api: Switch get_stats() to use SQLAlchemy<commit_after>'''The API calls'''

import StringIO
from flask import (
    jsonify,
    request,
    send_file,
)
import sqlalchemy
from sqlalchemy import (
    cast,
    desc as sql_desc,
    distinct,
    Float,
    func,
)
from . import app
from .helpers import get_db
from .search import (
    search_bgcs,
    create_cluster_csv,
    available_term_by_category,
)
from .models import (
    db,
    BgcType,
    BiosyntheticGeneCluster as Bgc,
    DnaSequence,
    Genome,
    Locus,
    Taxa,
    t_rel_clusters_types,
)
import sql

@app.route('/api/v1.0/version')
def get_version():
    '''display the API version'''
    from .version import __version__ as api_version
    ret = {
        'api': api_version,
        'sqlalchemy': sqlalchemy.__version__
    }
    return jsonify(ret)

@app.route('/api/v1.0/stats')
def get_stats():
    '''contents for the stats page'''
    num_clusters = Bgc.query.count()

    num_genomes = Genome.query.count()

    num_sequences = DnaSequence.query.count()

    clusters = []

    sub = db.session.query(t_rel_clusters_types.c.bgc_type_id, func.count(1).label('count')) \
                    .group_by(t_rel_clusters_types.c.bgc_type_id).subquery()
    ret = db.session.query(BgcType.term, BgcType.description, sub.c.count).join(sub) \
                    .order_by(sub.c.count.desc())
    for cluster in ret:
        clusters.append({'name': cluster.term, 'description': cluster.description, 'count': cluster.count})

    ret = db.session.query(Taxa.tax_id, Taxa.genus, Taxa.species, func.count(DnaSequence.acc).label('tax_count')) \
                    .join(Genome).join(DnaSequence) \
                    .group_by(Taxa.tax_id).order_by(sql_desc('tax_count')).limit(1).first()
    top_seq_taxon = ret.tax_id
    top_seq_taxon_count = ret.tax_count

    ret = db.session.query(Taxa.tax_id, Taxa.genus, Taxa.species, func.count(distinct(Bgc.bgc_id)).label('bgc_count'),
                           func.count(distinct(DnaSequence.acc)).label('seq_count'),
                           (cast(func.count(distinct(Bgc.bgc_id)), Float) / func.count(distinct(DnaSequence.acc))).label('clusters_per_seq')) \
                    .join(Genome).join(DnaSequence).join(Locus).join(Bgc) \
                    .group_by(Taxa.tax_id).order_by(sql_desc('clusters_per_seq')).limit(1).first()
    top_secmet_taxon = ret.tax_id
    top_secmet_species = ret.species
    top_secmet_taxon_count = ret.clusters_per_seq

    stats = {
        'num_clusters': num_clusters,
        'num_genomes': num_genomes,
        'num_sequences': num_sequences,
        'top_seq_taxon': top_seq_taxon,
        'top_seq_taxon_count': top_seq_taxon_count,
        'top_secmet_taxon': top_secmet_taxon,
        'top_secmet_taxon_count': top_secmet_taxon_count,
        'top_secmet_species': top_secmet_species,
        'clusters': clusters,
    }

    return jsonify(stats)

@app.route('/api/v1.0/tree/secmet')
def get_sec_met_tree():
    '''Get the jsTree structure for secondary metabolite clusters'''
    cur = get_db().cursor()
    cur.execute(sql.SECMET_TREE)
    ret = cur.fetchall()

    clusters = []
    types = {}

    for entry in ret:
        types[entry.term] = entry.description
        clusters.append({
            "id": "{}_c{}_{}".format(entry.acc, entry.cluster_number, entry.term),
            "parent": entry.term,
            "text": "{} {} Cluster {}".format(entry.species, entry.acc, entry.cluster_number),
            "type": "cluster",
        })

    for name, desc in types.iteritems():
        clusters.insert(0, {
            "id": name,
            "parent": "#",
            "text": desc,
            "state": {
                "disabled": True
            }
        })

    tree = clusters
    return jsonify(tree)

@app.route('/api/v1.0/tree/taxa')
def get_taxon_tree():
    '''Get the jsTree structure for all taxa'''
    tree_id = request.args.get('id', '1')

    cur = get_db().cursor()

    HANDLERS = {
        'superkingdom': get_phylum,
        'phylum': get_class,
        'class': get_order,
        'order': get_family,
        'family': get_genus,
        'genus': get_species,
    }

    if tree_id == '1':
        tree = get_superkingdom(cur)
    else:
        params = tree_id.split('_')
        taxlevel = params[0]
        params = params[1:]
        handler = HANDLERS.get(taxlevel, lambda x, y: [])
        tree = handler(cur, params)

    return jsonify(tree)

def get_superkingdom(cur):
    '''Get list of superkingdoms'''
    tree = []
    cur.execute(sql.TAXTREE_SUPERKINGOM)
    kingdoms = cur.fetchall()
    for kingdom in kingdoms:
        tree.append(_create_tree_node('superkingdom_{}'.format(kingdom[0].lower()),
                                      '#', kingdom[0]))

    return tree

def get_phylum(cur, params):
    '''Get list of phyla per kingdom'''
    tree = []
    cur.execute(sql.TAXTREE_PHYLUM, params)
    phyla = cur.fetchall()
    for phylum in phyla:
        id_list = params + [phylum[0].lower()]
        tree.append(_create_tree_node('phylum_{}'.format('_'.join(id_list)),
                                      'superkingdom_{}'.format('_'.join(params)),
                                      phylum[0]))

    return tree

def get_class(cur, params):
    '''Get list of classes per kingdom/phylum'''
    tree = []
    cur.execute(sql.TAXTREE_CLASS, params)
    classes = cur.fetchall()
    for cls in classes:
        id_list = params + [cls[0].lower()]
        tree.append(_create_tree_node('class_{}'.format('_'.join(id_list)),
                                      'phylum_{}'.format('_'.join(params)),
                                      cls[0]))
    return tree

def get_order(cur, params):
    '''Get list of oders per kingdom/phylum/class'''
    tree = []
    cur.execute(sql.TAXTREE_ORDER, params)
    orders = cur.fetchall()
    for order in orders:
        id_list = params + [order[0].lower()]
        tree.append(_create_tree_node('order_{}'.format('_'.join(id_list)),
                                      'class_{}'.format('_'.join(params)),
                                      order[0]))
    return tree

def get_family(cur, params):
    '''Get list of families per kingdom/phylum/class/order'''
    tree = []
    cur.execute(sql.TAXTREE_FAMILY, params)
    families = cur.fetchall()
    for family in families:
        id_list = params + [family[0].lower()]
        tree.append(_create_tree_node('family_{}'.format('_'.join(id_list)),
                                      'order_{}'.format('_'.join(params)),
                                      family[0]))
    return tree

def get_genus(cur, params):
    '''Get list of genera per kingdom/phylum/class/order/family'''
    tree = []
    cur.execute(sql.TAXTREE_GENUS, params)
    genera = cur.fetchall()
    for genus in genera:
        id_list = params + [genus[0].lower()]
        tree.append(_create_tree_node('genus_{}'.format('_'.join(id_list)),
                                      'family_{}'.format('_'.join(params)),
                                      genus[0]))
    return tree

def get_species(cur, params):
    '''Get list of species per kingdom/phylum/class/order/family/genus'''
    tree = []
    cur.execute(sql.TAXTREE_SPECIES, params)
    strains = cur.fetchall()
    for strain in strains:
        tree.append(_create_tree_node('{}'.format(strain.acc.lower()),
                                      'genus_{}'.format('_'.join(params)),
                                      '{} {}.{}'.format(strain.species, strain.acc, strain.version),
                                      disabled=False, leaf=True))

    return tree

@app.route('/api/v1.0/search', methods=['POST'])
def search():
    '''Handle searching the database'''
    search_string = request.json.get('search_string', '')
    try:
        offset = int(request.json.get('offset'))
    except TypeError:
        offset = 0

    try:
        paginate = int(request.json.get('paginate'))
    except TypeError:
        paginate = 50

    total_count, stats, found_bgcs = search_bgcs(search_string, offset=offset, paginate=paginate)

    result = {
        'total': total_count,
        'clusters': found_bgcs,
        'offset': offset,
        'paginate': paginate,
        'stats': stats,
    }

    return jsonify(result)

@app.route('/api/v1.0/export', methods=['POST'])
def export():
    '''Export the search results as CSV file'''
    search_string = request.json.get('search_string', '')
    _, _, found_bgcs = search_bgcs(search_string, mapfunc=create_cluster_csv)

    found_bgcs.insert(0, '#Species\tNCBI accession\tCluster number\tBGC type\tFrom\tTo\tMost similar known cluster\tSimilarity in %\tMIBiG BGC-ID\tResults URL')

    handle = StringIO.StringIO()
    for line in found_bgcs:
        handle.write('{}\n'.format(line))

    handle.seek(0)
    return send_file(handle, attachment_filename='asdb_search_results.csv', as_attachment=True)

@app.route('/api/v1.0/genome/<identifier>')
def show_genome(identifier):
    '''show information for a genome by identifier'''
    search_string = '[acc]{}'.format(identifier)
    _, _, found_bgcs = search_bgcs(search_string)

    return jsonify(found_bgcs)

@app.route('/api/v1.0/available/<category>/<term>')
def list_available(category, term):
    '''list available terms for a given category'''
    return jsonify(available_term_by_category(category, term))

def _create_tree_node(node_id, parent, text, disabled=True, leaf=False):
    '''create a jsTree node structure'''
    ret = {}
    ret['id'] = node_id
    ret['parent'] = parent
    ret['text'] = text
    if disabled:
        ret['state'] = {'disabled': True}
    if leaf:
        ret['type'] = 'strain'
    else:
        ret['children'] = True
    return ret
<|endoftext|>


<commit_before>import os
import sys

from core.models import Slice,Sliver,User,Flavor,Node,Image

class XOSNodeSelector(object):
    def __init__(self, user, mem_size=None, num_cpus=None, disk_size=None):
        self.user = user

    def get_allowed_nodes(self):
        # TODO: logic to get nodes that the user can use
        nodes = Node.objects.all()
        return nodes

    def get_nodes(self, quantity):
        nodes = self.get_allowed_nodes()
        # TODO: sort the nodes by some useful metric to pick the best one
        return nodes[:quantity]

<commit_msg>add hostname option to node selector<commit_after>import os
import sys

from core.models import Slice,Sliver,User,Flavor,Node,Image

class XOSNodeSelector(object):
    def __init__(self, user, mem_size=None, num_cpus=None, disk_size=None, hostname = None):
        self.user = user
        self.hostname = None

    def get_allowed_nodes(self):
        # TODO: logic to get nodes that the user can use
        nodes = Node.objects.all()

        if self.hostname:
            nodes = nodes.filter(name = self.hostname)

        return nodes

    def get_nodes(self, quantity):
        nodes = self.get_allowed_nodes()
        # TODO: sort the nodes by some useful metric to pick the best one
        return nodes[:quantity]

<|endoftext|>