<commit_before>"""Move files to local Galaxy upload directory and add to Galaxy Data Libraries.

Required configurable variables in upload:
  dir
"""
import collections
import os

from bcbio import utils
from bcbio.log import logger
from bcbio.upload import filesystem
from bcbio.pipeline import qcsummary

# Avoid bioblend import errors, raising at time of use
try:
    from bioblend.galaxy import GalaxyInstance
except ImportError:
    GalaxyInstance = None

def update_file(finfo, sample_info, config):
    """Update file in Galaxy data libraries.
    """
    if GalaxyInstance is None:
        raise ImportError("Could not import bioblend.galaxy")
    if "dir" not in config:
        raise ValueError("Galaxy upload requires `dir` parameter in config specifying the "
                         "shared filesystem path to move files to.")
    folder_name = "%s_%s" % (config["fc_date"], config["fc_name"])
    storage_dir = utils.safe_makedir(os.path.join(config["dir"], folder_name))
    if finfo.get("type") == "directory":
        storage_file = None
        if finfo.get("ext") == "qc":
            pdf_file = qcsummary.prep_pdf(finfo["path"], config)
            if pdf_file:
                finfo["path"] = pdf_file
                finfo["type"] = "pdf"
                storage_file = filesystem.copy_finfo(finfo, storage_dir, pass_uptodate=True)
    else:
        storage_file = filesystem.copy_finfo(finfo, storage_dir, pass_uptodate=True)
    if "galaxy_url" in config and "galaxy_api_key" in config:
        galaxy_url = config["galaxy_url"]
        if not galaxy_url.endswith("/"):
            galaxy_url += "/"
        gi = GalaxyInstance(galaxy_url, config["galaxy_api_key"])
    else:
        raise ValueError("Galaxy upload requires `galaxy_url` and `galaxy_api_key` in config")
    if storage_file and sample_info and not finfo.get("index", False):
        _to_datalibrary(storage_file, gi, folder_name, sample_info, config)

def _to_datalibrary(fname, gi, folder_name, sample_info, config):
    """Upload a file to a Galaxy data library in a project specific folder.
    """
    library = _get_library(gi, sample_info, config)
    libitems = gi.libraries.show_library(library.id, contents=True)
    folder = _get_folder(gi, folder_name, library, libitems)
    _file_to_folder(gi, fname, sample_info, libitems, library, folder)

def _file_to_folder(gi, fname, sample_info, libitems, library, folder):
    """Check if file exists on Galaxy, if not upload to specified folder.
    """
    full_name = os.path.join(folder["name"], os.path.basename(fname))

    # Handle VCF: Galaxy reports VCF files without the gzip extension
    file_type = "vcf_bgzip" if full_name.endswith(".vcf.gz") else "auto"
    if full_name.endswith(".vcf.gz"):
        full_name = full_name.replace(".vcf.gz", ".vcf")

    for item in libitems:
        if item["name"] == full_name:
            return item
    logger.info("Uploading to Galaxy library '%s': %s" % (library.name, full_name))
    return gi.libraries.upload_from_galaxy_filesystem(str(library.id), fname, folder_id=str(folder["id"]),
                                                      link_data_only="link_to_files",
                                                      dbkey=sample_info["genome_build"],
                                                      file_type=file_type,
                                                      roles=str(library.roles) if library.roles else None)

def _get_folder(gi, folder_name, library, libitems):
    """Retrieve or create a folder inside the library with the specified name.
    """
    for item in libitems:
        if item["type"] == "folder" and item["name"] == "/%s" % folder_name:
            return item
    return gi.libraries.create_folder(library.id, folder_name)[0]

GalaxyLibrary = collections.namedtuple("GalaxyLibrary", ["id", "name", "roles"])

def _get_library(gi, sample_info, config):
    """Retrieve the appropriate data library for the current user.
    """
    galaxy_lib = sample_info.get("galaxy_library",
                                 config.get("galaxy_library"))
    role = sample_info.get("galaxy_role",
                           config.get("galaxy_role"))
    if galaxy_lib:
        return _get_library_from_name(gi, galaxy_lib, role, sample_info, create=True)
    elif config.get("private_libs") or config.get("lab_association") or config.get("researcher"):
        return _library_from_nglims(gi, sample_info, config)
    else:
        raise ValueError("No Galaxy library specified for sample: %s" %
                         sample_info["description"])

def _get_library_from_name(gi, name, role, sample_info, create=False):
    for lib in gi.libraries.get_libraries():
        if lib["name"].lower() == name.lower() and not lib.get("deleted", False):
            return GalaxyLibrary(lib["id"], lib["name"], role)
    if create and name:
        logger.info("Creating Galaxy library: '%s'" % name)
        lib = gi.libraries.create_library(name)
        librole = str(gi.users.get_current_user()["id"] if not role else role)
        try:
            gi.libraries.set_library_permissions(str(lib["id"]), librole, librole, librole, librole)
        # XXX Returns error on Galaxy side but seems to work -- ugly
        except:
            pass
        return GalaxyLibrary(lib["id"], lib["name"], role)
    else:
        raise ValueError("Could not find Galaxy library matching '%s' for sample %s" %
                         (name, sample_info["description"]))

def _library_from_nglims(gi, sample_info, config):
    """Retrieve upload library from nglims specified user libraries.
    """
    names = [config.get(x, "").strip() for x in ["lab_association", "researcher"]
             if config.get(x)]
    for name in names:
        for ext in ["sequencing", "lab"]:
            check_name = "%s %s" % (name.split()[0], ext)
            try:
                return _get_library_from_name(gi, check_name, None, sample_info)
            except ValueError:
                pass
    check_names = set([x.lower() for x in names])
    for libname, role in config["private_libs"]:
        # Try to find library for lab or rsearcher
        if libname.lower() in check_names:
            return _get_library_from_name(gi, libname, role, sample_info)
    # default to first private library if available
    if len(config.get("private_libs", [])) > 0:
        libname, role = config["private_libs"][0]
        return _get_library_from_name(gi, libname, role, sample_info)
    # otherwise use the lab association or researcher name
    elif len(names) > 0:
        return _get_library_from_name(gi, names[0], None, sample_info, create=True)
    else:
        raise ValueError("Could not find Galaxy library for sample %s" % sample_info["description"])
<commit_msg>Provide resiliency to Galaxy API errors when uploading bcbio-nextgen results<commit_after>"""Move files to local Galaxy upload directory and add to Galaxy Data Libraries.

Required configurable variables in upload:
  dir
"""
import collections
import os
import time

import bioblend
import simplejson

from bcbio import utils
from bcbio.log import logger
from bcbio.upload import filesystem
from bcbio.pipeline import qcsummary

# Avoid bioblend import errors, raising at time of use
try:
    from bioblend.galaxy import GalaxyInstance
except ImportError:
    GalaxyInstance = None

def update_file(finfo, sample_info, config):
    """Update file in Galaxy data libraries.
    """
    if GalaxyInstance is None:
        raise ImportError("Could not import bioblend.galaxy")
    if "dir" not in config:
        raise ValueError("Galaxy upload requires `dir` parameter in config specifying the "
                         "shared filesystem path to move files to.")
    folder_name = "%s_%s" % (config["fc_date"], config["fc_name"])
    storage_dir = utils.safe_makedir(os.path.join(config["dir"], folder_name))
    if finfo.get("type") == "directory":
        storage_file = None
        if finfo.get("ext") == "qc":
            pdf_file = qcsummary.prep_pdf(finfo["path"], config)
            if pdf_file:
                finfo["path"] = pdf_file
                finfo["type"] = "pdf"
                storage_file = filesystem.copy_finfo(finfo, storage_dir, pass_uptodate=True)
    else:
        storage_file = filesystem.copy_finfo(finfo, storage_dir, pass_uptodate=True)
    if "galaxy_url" in config and "galaxy_api_key" in config:
        galaxy_url = config["galaxy_url"]
        if not galaxy_url.endswith("/"):
            galaxy_url += "/"
        gi = GalaxyInstance(galaxy_url, config["galaxy_api_key"])
    else:
        raise ValueError("Galaxy upload requires `galaxy_url` and `galaxy_api_key` in config")
    if storage_file and sample_info and not finfo.get("index", False):
        _to_datalibrary_safe(storage_file, gi, folder_name, sample_info, config)

def _to_datalibrary_safe(fname, gi, folder_name, sample_info, config):
    """Upload with retries for intermittent JSON failures.
    """
    num_tries = 0
    max_tries = 5
    while 1:
        try:
            _to_datalibrary(fname, gi, folder_name, sample_info, config)
            break
        except (simplejson.scanner.JSONDecodeError, bioblend.galaxy.client.ConnectionError) as e:
            num_tries += 1
            if num_tries > max_tries:
                raise
            print "Retrying upload, failed with:", str(e)
            time.sleep(5)

def _to_datalibrary(fname, gi, folder_name, sample_info, config):
    """Upload a file to a Galaxy data library in a project specific folder.
    """
    library = _get_library(gi, sample_info, config)
    libitems = gi.libraries.show_library(library.id, contents=True)
    folder = _get_folder(gi, folder_name, library, libitems)
    _file_to_folder(gi, fname, sample_info, libitems, library, folder)

def _file_to_folder(gi, fname, sample_info, libitems, library, folder):
    """Check if file exists on Galaxy, if not upload to specified folder.
    """
    full_name = os.path.join(folder["name"], os.path.basename(fname))

    # Handle VCF: Galaxy reports VCF files without the gzip extension
    file_type = "vcf_bgzip" if full_name.endswith(".vcf.gz") else "auto"
    if full_name.endswith(".vcf.gz"):
        full_name = full_name.replace(".vcf.gz", ".vcf")

    for item in libitems:
        if item["name"] == full_name:
            return item
    logger.info("Uploading to Galaxy library '%s': %s" % (library.name, full_name))
    return gi.libraries.upload_from_galaxy_filesystem(str(library.id), fname, folder_id=str(folder["id"]),
                                                      link_data_only="link_to_files",
                                                      dbkey=sample_info["genome_build"],
                                                      file_type=file_type,
                                                      roles=str(library.roles) if library.roles else None)

def _get_folder(gi, folder_name, library, libitems):
    """Retrieve or create a folder inside the library with the specified name.
    """
    for item in libitems:
        if item["type"] == "folder" and item["name"] == "/%s" % folder_name:
            return item
    return gi.libraries.create_folder(library.id, folder_name)[0]

GalaxyLibrary = collections.namedtuple("GalaxyLibrary", ["id", "name", "roles"])

def _get_library(gi, sample_info, config):
    """Retrieve the appropriate data library for the current user.
    """
    galaxy_lib = sample_info.get("galaxy_library",
                                 config.get("galaxy_library"))
    role = sample_info.get("galaxy_role",
                           config.get("galaxy_role"))
    if galaxy_lib:
        return _get_library_from_name(gi, galaxy_lib, role, sample_info, create=True)
    elif config.get("private_libs") or config.get("lab_association") or config.get("researcher"):
        return _library_from_nglims(gi, sample_info, config)
    else:
        raise ValueError("No Galaxy library specified for sample: %s" %
                         sample_info["description"])

def _get_library_from_name(gi, name, role, sample_info, create=False):
    for lib in gi.libraries.get_libraries():
        if lib["name"].lower() == name.lower() and not lib.get("deleted", False):
            return GalaxyLibrary(lib["id"], lib["name"], role)
    if create and name:
        logger.info("Creating Galaxy library: '%s'" % name)
        lib = gi.libraries.create_library(name)
        librole = str(gi.users.get_current_user()["id"] if not role else role)
        try:
            gi.libraries.set_library_permissions(str(lib["id"]), librole, librole, librole, librole)
        # XXX Returns error on Galaxy side but seems to work -- ugly
        except:
            pass
        return GalaxyLibrary(lib["id"], lib["name"], role)
    else:
        raise ValueError("Could not find Galaxy library matching '%s' for sample %s" %
                         (name, sample_info["description"]))

def _library_from_nglims(gi, sample_info, config):
    """Retrieve upload library from nglims specified user libraries.
    """
    names = [config.get(x, "").strip() for x in ["lab_association", "researcher"]
             if config.get(x)]
    for name in names:
        for ext in ["sequencing", "lab"]:
            check_name = "%s %s" % (name.split()[0], ext)
            try:
                return _get_library_from_name(gi, check_name, None, sample_info)
            except ValueError:
                pass
    check_names = set([x.lower() for x in names])
    for libname, role in config["private_libs"]:
        # Try to find library for lab or rsearcher
        if libname.lower() in check_names:
            return _get_library_from_name(gi, libname, role, sample_info)
    # default to first private library if available
    if len(config.get("private_libs", [])) > 0:
        libname, role = config["private_libs"][0]
        return _get_library_from_name(gi, libname, role, sample_info)
    # otherwise use the lab association or researcher name
    elif len(names) > 0:
        return _get_library_from_name(gi, names[0], None, sample_info, create=True)
    else:
        raise ValueError("Could not find Galaxy library for sample %s" % sample_info["description"])
<|endoftext|>


<commit_before>import datetime

from validate_email import validate_email

from bd_api import db
from bd_api.users.models import User
from bd_api.users.measurements.models import Measurement

class CommonUtils:

    def getUsername(userId):
        """Return username linked to user Id"""
        q = User.query.add_columns('username').filter_by(id = userId).first()
        if q:
            return q[1]
        else:
            return {'error': 'not found', 'userId': userId}

    def convertFromISODate(date):
        """Convert long ISO date to short if exists, else return None"""
        if date:
            try:
                datetime_object = datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%fZ')
            except ValueError:
                return date
            else:
                return datetime_object.strftime('%Y-%m-%d')
        else:
            return None

    def convertToISODate(date):
        if date:
            return date.isoformat()
        else:
            return None

    def countNumberOfRows(userId):
        numberOfMeasurements = Measurement.query.filter_by(user_id=userId).count()
        return(numberOfMeasurements)

class UserUtils:

    def validate_user_values(user_request):
        user_validation = {}
        if user_request.get('username'):
            if validate_username(user_request.get('username')):
                user_validation['username'] = validate_username(user_request.get('username'))['error']
        if user_request.get('password'):
            if validateString('password', user_request.get('password'), 80):
                user_validation['password'] = validateString('password', user_request.get('password'), 120)['error']
        if user_request.get('email'):
            if isinstance(user_request.get('email'), str):
                if not validate_email(user_request.get('email')):
                    user_validation['email'] = 'invalid email %s' % user_request.get('email')
            else:
                user_validation['email'] = 'invalid email %s (%s)' % (user_request.get('email'), pythonTypeToJSONType(user_request.get('email')))
        if user_request.get('firstName'):
            if validateString('firstName', user_request.get('firstName'), 80):
                user_validation['firstName'] = validateString('firstName', user_request.get('firstName'), 80)['error']
        if user_request.get('lastName'):
            if validateString('lastName', user_request.get('lastName'), 80):
                user_validation['lastName'] = validateString('lastName', user_request.get('lastName'), 80)['error']
        if user_request.get('gender'):
            if validateGender(user_request.get('gender')):
                user_validation['gender'] = validateGender(user_request.get('gender'))['error']
        if user_request.get('dateOfBirth'):
            if validateDate(None, 'dateOfBirth', user_request.get('dateOfBirth')):
                user_validation['dateOfBirth'] = validateDate(None, 'dateOfBirth', user_request.get('dateOfBirth'))['error']
        if user_request.get('public'):
            if validateBoolean('public', user_request.get('public')):
                user_validation['public'] = validateBoolean('public', user_request.get('public'))['error']
        if user_validation:
            user_validation['error'] = 'Invalid value(s)'
        return user_validation

    def userExists(userId):
        return User.query.filter_by(id = userId).first()

    def getUserIdAndRole(username):
        """Return user Id linked to username"""
        q = User.query.add_columns('id').add_columns('role').filter_by(username=username).first()
        if q:
            return q[1], q[2]
        else:
            return False

    def checkIfPublicUser(userId):
        """Return None if user with userId is not public"""
        return User.query.filter_by(public = True).filter_by(id = userId).first()

class MeasurementUtils:

    def validate_measurement_values(userId, measurement_request):
        measurement_validation = {}
        for key, value in measurement_request.items():
            if key == 'measurementDate':
                if validateDate(userId, key, value):
                    measurement_validation[key] = validateDate(userId, key, value)['error']
            elif key in Measurement.measurement_keys:
                if validateNumber(key, value):
                    measurement_validation[key] = validateNumber(key, value)['error']
        return measurement_validation

def validate_username(username):
    """Return None if username is string 1 to 80 characters and username is not in use"""
    if not username or not isinstance(username, str) or len(username) > 80:
        return {'error': 'invalid username: %s (%s), valid value string 1 to 80 characters, mandatory' % (username, pythonTypeToJSONType(username))}
    q = User.query.filter_by(username = username).first()
    if q:
        return {'error': 'username %s is in use' % username}
    else:
        return None

def validateNumber(key, value):
    """Return None if value data type is None, Int or Float"""
    if value is None or isinstance(value, (int, float)) and not isinstance(value, bool):
        return None
    else:
        return {'error': 'invalid value: %s (%s), valid values number/null' % (value, pythonTypeToJSONType(value))}

def validateString(key, value, maxLenght):
    """Return None if value is None or string including 1 to 80 characters"""
    if value is None or isinstance(value, str) and len(value) <= maxLenght and len(value) > 0:
        return None
    else:
        return {'error': 'invalid value: %s (%s)' % (value, pythonTypeToJSONType(value))}

def validateGender(gender):
    """Return True if gender equals 'male', 'female' or None"""
    if gender == "male" or gender == "female" or gender == None:
        return None
    else:
        return {'error': 'invalid value: %s (%s), valid values male/female/null' % (gender, pythonTypeToJSONType(gender))}

def validateBoolean(key, value):
    """Return None if value data type is None or boolean"""
    if value is None or isinstance(value, bool):
        return None
    else:
        return {'error': 'invalid value: %s (%s), valid values boolean/null' % (value, pythonTypeToJSONType(value))}

def validateDate(userId, key, date):
    """Return None if date is in ISO 8601 format and user doesn't already have measurements with that date"""
    if key == 'dateOfBirth' and date == None:
        return None

    if not isinstance(date, str):
        return {'error': 'invalid value: %s (%s), valid value date in ISO 8601 format' % (date, pythonTypeToJSONType(date))}

    if 'T' in date:
        try:
            datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%fZ')
        except TypeError:
            return {'error': 'invalid value: %s (%s), valid value date in ISO 8601 format' % (date, pythonTypeToJSONType(date))}
        except ValueError:
            return {'error': 'invalid value: %s, valid value date in ISO 8601 format' % date}
    else:
        try:
            datetime.datetime.strptime(date, '%Y-%m-%d')
        except TypeError:
            return {'error': 'invalid value: %s (%s), valid value date in ISO 8601 format' % (date, pythonTypeToJSONType(date))}
        except ValueError:
            return {'error': 'invalid value: %s, valid value date in ISO 8601 format' % date}

    if userId:
        q = Measurement.query.add_columns('id').filter_by(user_id = userId).filter_by(measurementDate = date).first()
        if q:
            return {'error': 'duplicate value found for date %s (data id %s)' % (date, q[1])}
        else:
            return None

def pythonTypeToJSONType(value):
    """Convert Python data type to JSON data type"""
    if isinstance(value, dict): return 'object'
    elif isinstance(value, (list, tuple)): return 'array'
    elif isinstance(value, str): return 'string'
    elif isinstance(value, bool): return 'boolean'
    elif isinstance(value, (int, float)): return 'number'
    elif value is None: return 'null'
    else: return 'unknown'
<commit_msg>util to check whether data id for user exists or not<commit_after>import datetime

from validate_email import validate_email

from bd_api import db
from bd_api.users.models import User
from bd_api.users.measurements.models import Measurement

class CommonUtils:

    def getUsername(userId):
        """Return username linked to user Id"""
        q = User.query.add_columns('username').filter_by(id = userId).first()
        if q:
            return q[1]
        else:
            return {'error': 'not found', 'userId': userId}

    def convertFromISODate(date):
        """Convert long ISO date to short if exists, else return None"""
        if date:
            try:
                datetime_object = datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%fZ')
            except ValueError:
                return date
            else:
                return datetime_object.strftime('%Y-%m-%d')
        else:
            return None

    def convertToISODate(date):
        if date:
            return date.isoformat()
        else:
            return None

    def countNumberOfRows(userId):
        numberOfMeasurements = Measurement.query.filter_by(user_id=userId).count()
        return(numberOfMeasurements)

class UserUtils:

    def validate_user_values(user_request):
        user_validation = {}
        if user_request.get('username'):
            if validate_username(user_request.get('username')):
                user_validation['username'] = validate_username(user_request.get('username'))['error']
        if user_request.get('password'):
            if validateString('password', user_request.get('password'), 80):
                user_validation['password'] = validateString('password', user_request.get('password'), 120)['error']
        if user_request.get('email'):
            if isinstance(user_request.get('email'), str):
                if not validate_email(user_request.get('email')):
                    user_validation['email'] = 'invalid email %s' % user_request.get('email')
            else:
                user_validation['email'] = 'invalid email %s (%s)' % (user_request.get('email'), pythonTypeToJSONType(user_request.get('email')))
        if user_request.get('firstName'):
            if validateString('firstName', user_request.get('firstName'), 80):
                user_validation['firstName'] = validateString('firstName', user_request.get('firstName'), 80)['error']
        if user_request.get('lastName'):
            if validateString('lastName', user_request.get('lastName'), 80):
                user_validation['lastName'] = validateString('lastName', user_request.get('lastName'), 80)['error']
        if user_request.get('gender'):
            if validateGender(user_request.get('gender')):
                user_validation['gender'] = validateGender(user_request.get('gender'))['error']
        if user_request.get('dateOfBirth'):
            if validateDate(None, 'dateOfBirth', user_request.get('dateOfBirth')):
                user_validation['dateOfBirth'] = validateDate(None, 'dateOfBirth', user_request.get('dateOfBirth'))['error']
        if user_request.get('public'):
            if validateBoolean('public', user_request.get('public')):
                user_validation['public'] = validateBoolean('public', user_request.get('public'))['error']
        if user_validation:
            user_validation['error'] = 'Invalid value(s)'
        return user_validation

    def userExists(userId):
        return User.query.filter_by(id = userId).first()

    def getUserIdAndRole(username):
        """Return user Id linked to username"""
        q = User.query.add_columns('id').add_columns('role').filter_by(username=username).first()
        if q:
            return q[1], q[2]
        else:
            return False

    def checkIfPublicUser(userId):
        """Return None if user with userId is not public"""
        return User.query.filter_by(public = True).filter_by(id = userId).first()

class MeasurementUtils:

    def measurement_exists(userId, dataId):
        return Measurement.query.filter_by(id = dataId).filter_by(user_id = userId).first()

    def validate_measurement_values(userId, measurement_request):
        measurement_validation = {}
        for key, value in measurement_request.items():
            if key == 'measurementDate':
                if validateDate(userId, key, value):
                    measurement_validation[key] = validateDate(userId, key, value)['error']
            elif key in Measurement.measurement_keys:
                if validateNumber(key, value):
                    measurement_validation[key] = validateNumber(key, value)['error']
        return measurement_validation

def validate_username(username):
    """Return None if username is string 1 to 80 characters and username is not in use"""
    if not username or not isinstance(username, str) or len(username) > 80:
        return {'error': 'invalid username: %s (%s), valid value string 1 to 80 characters, mandatory' % (username, pythonTypeToJSONType(username))}
    q = User.query.filter_by(username = username).first()
    if q:
        return {'error': 'username %s is in use' % username}
    else:
        return None

def validateNumber(key, value):
    """Return None if value data type is None, Int or Float"""
    if value is None or isinstance(value, (int, float)) and not isinstance(value, bool):
        return None
    else:
        return {'error': 'invalid value: %s (%s), valid values number/null' % (value, pythonTypeToJSONType(value))}

def validateString(key, value, maxLenght):
    """Return None if value is None or string including 1 to 80 characters"""
    if value is None or isinstance(value, str) and len(value) <= maxLenght and len(value) > 0:
        return None
    else:
        return {'error': 'invalid value: %s (%s)' % (value, pythonTypeToJSONType(value))}

def validateGender(gender):
    """Return True if gender equals 'male', 'female' or None"""
    if gender == "male" or gender == "female" or gender == None:
        return None
    else:
        return {'error': 'invalid value: %s (%s), valid values male/female/null' % (gender, pythonTypeToJSONType(gender))}

def validateBoolean(key, value):
    """Return None if value data type is None or boolean"""
    if value is None or isinstance(value, bool):
        return None
    else:
        return {'error': 'invalid value: %s (%s), valid values boolean/null' % (value, pythonTypeToJSONType(value))}

def validateDate(userId, key, date):
    """Return None if date is in ISO 8601 format and user doesn't already have measurements with that date"""
    if key == 'dateOfBirth' and date == None:
        return None

    if not isinstance(date, str):
        return {'error': 'invalid value: %s (%s), valid value date in ISO 8601 format' % (date, pythonTypeToJSONType(date))}

    if 'T' in date:
        try:
            datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%fZ')
        except TypeError:
            return {'error': 'invalid value: %s (%s), valid value date in ISO 8601 format' % (date, pythonTypeToJSONType(date))}
        except ValueError:
            return {'error': 'invalid value: %s, valid value date in ISO 8601 format' % date}
    else:
        try:
            datetime.datetime.strptime(date, '%Y-%m-%d')
        except TypeError:
            return {'error': 'invalid value: %s (%s), valid value date in ISO 8601 format' % (date, pythonTypeToJSONType(date))}
        except ValueError:
            return {'error': 'invalid value: %s, valid value date in ISO 8601 format' % date}

    if userId:
        q = Measurement.query.add_columns('id').filter_by(user_id = userId).filter_by(measurementDate = date).first()
        if q:
            return {'error': 'duplicate value found for date %s (data id %s)' % (date, q[1])}
        else:
            return None

def pythonTypeToJSONType(value):
    """Convert Python data type to JSON data type"""
    if isinstance(value, dict): return 'object'
    elif isinstance(value, (list, tuple)): return 'array'
    elif isinstance(value, str): return 'string'
    elif isinstance(value, bool): return 'boolean'
    elif isinstance(value, (int, float)): return 'number'
    elif value is None: return 'null'
    else: return 'unknown'
<|endoftext|>


<commit_before>import json
import sys

from behave.configuration import Configuration
from behave.formatter.base import StreamOpener
from six import StringIO

from bdd_tester.runner import DQRunner

def bdd_tester(filepath, features, **kwargs):
    default_output_path = 'output'

    # we'll add the behave args to this list
    command_args = []

    # disable the default summary
    command_args.append('--no-summary')

    # don't capture stdout. It doesn't matter, really...
    # but we don't need this (since we do it ourselves)
    command_args.append('--no-capture')

    # don't capture stderr either
    command_args.append('--no-capture-stderr')

    # but DO capture logging
    command_args.append('--logcapture')
    command_args += ['--logging-format', '%(levelname)s:%(message)s']
    command_args += ['--logging-level', kwargs.get('logging_level')]

    # declare some custom formatters
    formatters = [
        'bdd_tester.formatters:DQSummaryFormatter',
        'bdd_tester.formatters:DQLogFormatter',
    ]
    for formatter in formatters:
        command_args += ['--format', formatter]

    # pass stuff to behave via user-defined variables
    if kwargs.get('today'):
        command_args += ['--define', 'today=' + kwargs.get('today')]
    if kwargs.get('output_path'):
        output_path = kwargs.get('output_path')
    else:
        output_path = default_output_path
    command_args += ['--define', 'output_path=' + output_path]

    save_summary = kwargs.get('save_summary', False)
    if save_summary:
        # specify the summary formatter output filename
        command_args += ['--outfile', 'summary.output']

    # specify the location of the test files (features)
    command_args += features

    try:
        # create a config instance
        config = Configuration(command_args, load_config=False)

        if not save_summary:
            summary = StringIO()
            config.outputs.insert(0, StreamOpener(stream=summary))

        # construct the runner!
        runner = DQRunner(config)

        # get this show on the road
        runner.start(filepath)

        if not save_summary:
            # dump captured output
            return json.loads(summary.getvalue())
    except:
        if not save_summary:
            print(summary.getvalue())
        raise
<commit_msg>Comments<commit_after>import json
import sys

from behave.configuration import Configuration
from behave.formatter.base import StreamOpener
from six import StringIO

from bdd_tester.runner import DQRunner

def bdd_tester(filepath, features, **kwargs):
    default_output_path = 'output'

    # we'll add the behave args to this list
    command_args = []

    # disable the default summary
    command_args.append('--no-summary')

    # don't capture stdout. It doesn't matter, really...
    # but we don't need this (since we do it ourselves)
    command_args.append('--no-capture')

    # don't capture stderr either
    command_args.append('--no-capture-stderr')

    # but DO capture logging
    command_args.append('--logcapture')
    command_args += ['--logging-format', '%(levelname)s:%(message)s']
    command_args += ['--logging-level', kwargs.get('logging_level')]

    # declare some custom formatters
    formatters = [
        'bdd_tester.formatters:DQSummaryFormatter',
        'bdd_tester.formatters:DQLogFormatter',
    ]
    for formatter in formatters:
        command_args += ['--format', formatter]

    # pass stuff to behave via user-defined variables
    if kwargs.get('today'):
        command_args += ['--define', 'today=' + kwargs.get('today')]
    if kwargs.get('output_path'):
        output_path = kwargs.get('output_path')
    else:
        output_path = default_output_path
    command_args += ['--define', 'output_path=' + output_path]

    save_summary = kwargs.get('save_summary', False)
    if save_summary:
        # specify the summary formatter output filename
        command_args += ['--outfile', 'summary.output']

    # specify the location of the test files (features)
    command_args += features

    try:
        # create a config instance
        config = Configuration(command_args, load_config=False)

        if not save_summary:
            # summary formatter should output to a buffer
            summary = StringIO()
            config.outputs.insert(0, StreamOpener(stream=summary))

        # construct the runner!
        runner = DQRunner(config)

        # get this show on the road
        runner.start(filepath)

        if not save_summary:
            # dump summary formatter buffer
            return json.loads(summary.getvalue())
    except:
        if not save_summary:
            # something went wrong; dump summary formatter buffer
            print(summary.getvalue())
        raise
<|endoftext|>


<commit_before>import re

from coalib.bearlib.abstractions.Lint import Lint
from coalib.bears.LocalBear import LocalBear

class GoTypeBear(LocalBear, Lint):
    executable = 'gotype'
    arguments = "-e {filename}"
    use_stderr = True
    output_regex = re.compile(
        r'(?P<file_name>.*):(?P<line>\d+):(?P<column>\d+):\s*(?P<message>.*)')
    LANGUAGES = "Go"

    def run(self, filename, file):
        '''
        Checks the code using ``gotype``. This will run gotype over each file
        seperately.
        '''
        return self.lint(filename)
<commit_msg>GoTypeBear: Refactor to Linter v2 API<commit_after>from coalib.bearlib.abstractions.Linter import linter

@linter(executable='gotype',
        use_stdout=False,
        use_stderr=True,
        output_format='regex',
        output_regex=r'.+:(?P<line>\d+):(?P<column>\d+): *(?P<message>.*)')
class GoTypeBear:
    """
    Checks the code using ``gotype``. This will run ``gotype`` over each file
    separately.
    """
    LANGUAGES = "Go"

    @staticmethod
    def create_arguments(filename, file, config_file):
        return '-e', filename
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.core.management.base import BaseCommand
from django.db.models import Q

from digest.management.commands.create_dataset import create_dataset
from digest.models import Item

class Command(BaseCommand):
    help = 'Create dataset'

    def handle(self, *args, **options):
        """
        Основной метод - точка входа
        """

        query = Q()

        urls = [
            'allmychanges.com',
            'stackoverflow.com',
        ]
        for entry in urls:
            query = query | Q(link__contains=entry)

        create_dataset(Item.objects.exclude(query).order_by('?'), 'items.json')
<commit_msg>Update export items command for dataset<commit_after># -*- coding: utf-8 -*-
from django.core.management.base import BaseCommand
from django.db.models import Q

from digest.management.commands.create_dataset import create_dataset
from digest.models import Item

class Command(BaseCommand):
    help = 'Create dataset'

    def handle(self, *args, **options):
        """
        Основной метод - точка входа
        """

        query = Q()

        urls = [
            'allmychanges.com',
            'stackoverflow.com',
        ]
        for entry in urls:
            query = query | Q(link__contains=entry)

        # TODO make raw sql
        active_news = Item.objects.filter(status='active').exclude(query)
        links = active_news.all().values_list('link', flat=True).distinct()
        non_active_news = Item.objects.exclude(link__in=links).exclude(query)

        items_ids = list(active_news.values_list('id', flat=True))
        items_ids.extend(non_active_news.values_list('id', flat=True))
        items_ids = list(set(items_ids))

        items = Item.objects.filter(id__in=items_ids)

        create_dataset(items, 'items.json')
<|endoftext|>


<commit_before>"""
The NetworKit benchmark

This module implements a comprehensive benchmark of NetworKit's analytics algorithms

"""

import pandas
import sys
import warnings
import math
import os
import numpy
import matplotlib.pyplot as plt
import seaborn
from time import gmtime, strftime
import signal

import networkit

from util import *
import nk
#import nx
#import ig
#import gt

# helper function

def averageRuns(df, groupby=["graph"]):
    """ Average running time, modularity, edges per second and number of clusters over multiple runs"""
    df = df.groupby(groupby, as_index=False).mean()
    df = df.sort("m", ascending=True)    # sort by graph size
    return df

def graphMeta(graphNames, graphDir):
    meta = []
    for name in graphNames:
        info("loading {name}".format(**locals()))
        G = networkit.readGraph(os.path.join(graphDir, "{0}.gml.graph".format(name)), networkit.Format.GML)
        (n, m) = networkit.properties.size(G)
        meta.append({"name" : name, "n" : n, "m" : m})
    info("done")
    return pandas.DataFrame(meta, columns=["name", "n", "m"])

def saveData(df, name):
    df.to_csv(os.path.join(bench.dataPath, "{0}.csv".format(name)), sep="\t")

# timeout

class Timeout(Exception):
    pass

def timeoutHandler(signum, frame):
    error("timeout")
    raise Timeout()

class bFail:
    name = "Fail"

    def run(self, G):
        raise Exception("FAIL!")

# Plots

## plot settings

seaborn.set_style("whitegrid")

### Colors
lightred = seaborn.xkcd_rgb["red"]
darkred = seaborn.xkcd_rgb["crimson"]
green = seaborn.xkcd_rgb["teal"]
orange = seaborn.xkcd_rgb["bright orange"]

# plot functions

def timePlot(data, size=(6,3)):
    pos = numpy.arange(len(data))+.5    # the bar centers on the y axis
    labels = list(data["graph"])
    plt.figure(figsize=size)
    plt.xscale("symlog")
    plt.barh(pos, data["time"], align='center', height=0.25, color=lightred)    # notice the 'height' argument
    plt.yticks(pos, labels)
    plt.gca().xaxis.set_minor_locator(plt.LogLocator(subs=[0,1,2,3,4,5,6,7,8,9,10]))
    #gca().xaxis.set_minor_formatter(FormatStrFormatter("%.2f"))
    plt.xlabel("time [s]")
    plt.grid(True)

def epsPlot(data, size=(6,3)):
    pos = numpy.arange(len(data))+.5    # the bar centers on the y axis
    labels = list(data["graph"])
    plt.figure(figsize=size)
    plt.xscale("log")
    plt.barh(pos, data["eps"], align='center', height=0.25, color=green)    # notice the 'height' argument
    plt.yticks(pos, labels)
    plt.gca().xaxis.set_minor_locator(plt.LogLocator(subs=[0,1,2,3,4,5,6,7,8,9,10]))
    #gca().xaxis.set_minor_formatter(FormatStrFormatter("%.2f"))
    plt.xlabel("edges / s")
    plt.grid(True)

def graphPlot(data, size=None):
    pos = arange(len(data))    # the bar centers on the y axis
    plt.figure(figsize=(5,3.3))
    sizes = data["m"]
    labels = list(data["graph"])
    #barh(pos, data["n"], log=True, align='center', height=0.4, color="darkgrey")    # notice the 'height' argument
    plt.barh(pos, sizes, log=True, align='center', height=0.4, color="lightblue", label="m")    # notice the 'height' argument
    plt.yticks(pos, labels)
    plt.xlabel("number of edges")
    plt.grid(True)

def barPlot(data, labels, x_label="", y_label="", size=None, color="b", transparency=1.0, scale="linear"):
    pos = numpy.arange(len(data))+.5    # the bar centers on the y axis
    plt.figure(figsize=size)
    plt.xscale(scale)
    plt.barh(pos, data, align='center', height=0.25, color=color, alpha=transparency)    # notice the 'height' argumen
    plt.yticks(pos, labels)
    plt.xlabel(x_label)
    plt.ylabel(y_label)
    plt.grid(True)

class Bench:

    """
    A Bench objects represents a benchmarking session. It is responsible
    for running the benchmarks, collecting and storing the resulting data.

    """

    def __init__(self, graphDir, defaultGraphs, outDir, save=True, nRuns=5, cacheGraphs=False, timeout=None):
        self.defaultGraphs = defaultGraphs
        self.nRuns = nRuns  # default number of runs for each algo
        self.graphDir = graphDir
        # store result data of benchmarks
        self.data = {}
        self.loadTimes = []
        self.outDir = outDir
        self.save = save  # store data frames on disk if true
        # create output directory if it does not exist
        if self.save:
            if not os.path.isdir(self.outDir):
                os.mkdir(self.outDir)
            self.outDataDir = os.path.join(self.outDir, "data")
            self.plotDir = os.path.join(self.outDir, "plots")
            if not os.path.isdir(self.outDataDir):
                os.mkdir(self.outDataDir)
            if not os.path.isdir(self.plotDir):
                os.mkdir(self.plotDir)
            # log file
            self.logPath = os.path.join(self.outDir, "log.txt")
        # graph cache
        self.cacheGraphs = cacheGraphs
        self.graphCache = {}
        #timeout
        self.timeout = timeout

    def getGraph(self, graphName, algo):
        """" Get the graph from disk or from in-memory cache"""
        key = algo.framework + graphName
        if key in self.graphCache:
            self.info("getting {0} from cache".format(graphName))
            G = self.graphCache[key]
            return G
        else:
            self.info("loading {0}".format(graphName))
            with Timer() as t:
                G = algo.loadGraph(os.path.join(self.graphDir, "{0}.gml.graph".format(graphName)))
            self.loadTimes.append({"framework" : algo.framework, "graph" : graphName, "time": t.elapsed})
            if self.cacheGraphs:
                self.graphCache[key] = G
            return G

    def getSize(self, G):
        if isinstance(G, networkit.Graph):
            return G.size()
        else:
            raise Error("cannot get graph size - unknown object type")

    def clearCache(self):
        """ Delete all stored graphs to free memory """
        del self.graphCache
        self.graphCache = {}

    def algoBenchmark(self, algo, graphs=None, nRuns=None, timeout=None):
        """ Run a kernel represented by an algorithm benchmark object """
        # set the defaults
        if nRuns is None:
            nRuns = self.nRuns  # lets argument override the default nRuns
        if graphs is None:
            graphs = self.defaultGraphs
        if timeout is None:
            timeout = self.timeout

        self.info("benchmarking {algo.name}".format(**locals()))
        table = []  # list of dictionaries, to be converted to a DataFrame

        for graphName in graphs:
            try:
                G = self.getGraph(graphName, algo)
                (n, m) = self.getSize(G)
                try:
                    self.info("running {algo.name} {nRuns}x on {graphName}".format(**locals()))
                    for i in range(nRuns):
                        row = {}    # benchmark data row
                        row["algo"] = algo.name
                        row["graph"] = graphName
                        row["m"] = m
                        try: # timeout
                            if timeout:
                                signal.signal(signal.SIGALRM, timeoutHandler)
                                signal.alarm(int(timeout * 60))  # timeout in seconds
                            with Timer() as t:
                                result = algo.run(G)
                            self.info("took {0} s".format(t.elapsed))
                            # store data
                            row["time"] = t.elapsed
                            row["eps"] =  m / t.elapsed  # calculate edges per second
                            row["result"] = result
                        except Timeout as tx:
                            self.error("{algo.name} timed out after {timeout} minutes".format(**locals()))
                            row["time"] = None
                            row["eps"] = None
                            row["result"] = None
                        finally:
                            table.append(row)
                            signal.alarm(int(1e9))    # in any case, cancel the timeout alarm by setting it to a ridiculously high time
                except Exception as ex:
                    self.error("algorithm {algo.name} failed with exception: {ex}".format(**locals()))
            except Exception as ex:
                self.error("loading graph {graphName} failed with exception: {ex}".format(**locals()))

        df = pandas.DataFrame(table)
        df.sort("m")    # sort by number of edges
        self.data[algo.name] = df
        # store data frame on disk
        if self.save:
            df.to_csv(os.path.join(self.outDataDir, "{algo.name}.csv".format(**locals())))
        self.log("Done")

    def log(self, message):
        """ Write a message to the logfile"""
        with open(self.logPath, "a") as logfile:
            logfile.write("{0}: {1}\n".format(strftime("%Y-%m-%d %H:%M:%S", gmtime()), message))

    def info(self, message):
        print(message)
        sys.stdout.flush()
        self.log(message)

    def error(self, message):
        print(message)
        sys.stdout.flush()
        self.log(message)

    def debug(self, message):
        pass
        # print(message)

    def timePlot(self, algoName):
        timePlot(averageRuns(self.data[algoName]))
        if self.save:
            plt.savefig(os.path.join(self.plotDir, "{algoName}-time.pdf".format(**locals())), bbox_inches="tight")

    def epsPlot(self, algoName):
        epsPlot(averageRuns(self.data[algoName]))
        if self.save:
            plt.savefig(os.path.join(self.plotDir, "{algoName}-eps.pdf".format(**locals())), bbox_inches="tight")

    def graphPlot(self):
        epsPlot(averageRuns(self.data))
        if self.save:
            plt.savefig(os.path.join(self.plotDir, "graphs.pdf".format(**locals())), bbox_inches="tight")

    def plot(self, algoName):
        print(algoName)
        self.timePlot(algoName)
        self.epsPlot(algoName)

    def plotSummary(self, algoNames=None, figsize=None):
        if algoNames is None:
            algoNames = list(self.data.keys())
        epsSummary = pandas.DataFrame()
        for (algoName, algoData) in self.data.items():
            if algoName in algoNames:
                epsSummary[algoName] = pandas.Series(self.data[algoName]["eps"])

        # data frame
        self.epsSummary = epsSummary
        if self.save:
            epsSummary.to_csv(os.path.join(self.outDataDir, "epsSummary.csv".format(**locals())))
        # plot
        if figsize:
            plt.figure(figsize=figsize)
        plt.gca().xaxis.get_major_formatter().set_powerlimits((3, 3))
        plt.xscale("log")
        plt.xlabel("edges/s")
        seaborn.boxplot(epsSummary, linewidth=1.5, width=.25, color=green, vert=False)
        if self.save:
            plt.savefig(os.path.join(self.plotDir, "epsSummary.pdf".format(**locals())), bbox_inches="tight")

    def plotAll(self):
        for key in self.data.keys():
            self.plot(key)

    def getLoadTimes(self):
        """ Get input times for graphs"""
        return pandas.DataFrame(self.loadTimes)

    def generatorBenchmark(self, generator, argtuples, nRuns=None, timeout=None):
        """ Run a kernel represented by an algorithm benchmark object """
        # set the defaults
        if nRuns is None:
            nRuns = self.nRuns  # lets argument override the default nRuns
        if timeout is None:
            timeout = self.timeout

        genName = str(generator)

        self.info("benchmarking {genName}".format(**locals()))
        table = []  # list of dictionaries, to be converted to a DataFrame

        for param in argtuples:
            try:
                gen = generator(*param)
                row = {}    # benchmark data row
                row["graph"] = str(param)
                try:
                    self.info("running {genName} {nRuns} times".format(**locals()))
                    for i in range(nRuns):
                        row["algo"] = genName
                        try: # timeout
                            result = None
                            if timeout:
                                signal.signal(signal.SIGALRM, timeoutHandler)
                                signal.alarm(int(timeout * 60))  # timeout in seconds
                            with Timer() as t:
                                result = gen.generate()
                            self.debug("took {0} s".format(t.elapsed))
                            # store data
                            row["m"] = result.numberOfNodes()
                            row["time"] = t.elapsed
                            row["result"] = result
                        except Timeout as tx:
                            self.error("{genName} timed out after {timeout} minutes".format(**locals()))
                            row["time"] = None
                            row["result"] = None
                        finally:
                            table.append(row)
                            signal.alarm(int(1e9))    # in any case, cancel the timeout alarm by setting it to a ridiculously high time
                except Exception as ex:
                    self.error("generator {genName} failed with exception: {ex}".format(**locals()))
            except Exception as ex:
                self.error("initializing generator {genName} failed with exception: {ex}".format(**locals()))

        df = pandas.DataFrame(table)
        self.data[genName] = df
        # store data frame on disk
        if self.save:
            df.to_csv(os.path.join(self.outDataDir, "{genName}.csv".format(**locals())))

# - generators
# 	- Erdös-Renyi (generators.ErdosRenyiGenerator)
# 	- Barabasi-Albert (generators.BarabasiAlbertGenerator)
# 	- Chung-Lu (generators.ChungLuGenerator)
# 	- RMAT (generators.RmatGenerator)
<commit_msg>graph load times in benchmark<commit_after>"""
The NetworKit benchmark

This module implements a comprehensive benchmark of NetworKit's analytics algorithms

"""

import pandas
import sys
import warnings
import math
import os
import numpy
import matplotlib.pyplot as plt
import seaborn
from time import gmtime, strftime
import signal

import networkit

from util import *
import nk
#import nx
#import ig
#import gt

# helper function

def averageRuns(df, groupby=["graph"]):
    """ Average running time, modularity, edges per second and number of clusters over multiple runs"""
    df = df.groupby(groupby, as_index=False).mean()
    df = df.sort("m", ascending=True)    # sort by graph size
    return df

def graphMeta(graphNames, graphDir):
    meta = []
    for name in graphNames:
        info("loading {name}".format(**locals()))
        G = networkit.readGraph(os.path.join(graphDir, "{0}.gml.graph".format(name)), networkit.Format.GML)
        (n, m) = networkit.properties.size(G)
        meta.append({"name" : name, "n" : n, "m" : m})
    info("done")
    return pandas.DataFrame(meta, columns=["name", "n", "m"])

def saveData(df, name):
    df.to_csv(os.path.join(bench.dataPath, "{0}.csv".format(name)), sep="\t")

# timeout

class Timeout(Exception):
    pass

def timeoutHandler(signum, frame):
    error("timeout")
    raise Timeout()

class bFail:
    name = "Fail"

    def run(self, G):
        raise Exception("FAIL!")

# Plots

## plot settings

seaborn.set_style("whitegrid")

### Colors
lightred = seaborn.xkcd_rgb["red"]
darkred = seaborn.xkcd_rgb["crimson"]
green = seaborn.xkcd_rgb["teal"]
orange = seaborn.xkcd_rgb["bright orange"]

# plot functions

def timePlot(data, size=(6,3)):
    pos = numpy.arange(len(data))+.5    # the bar centers on the y axis
    labels = list(data["graph"])
    plt.figure(figsize=size)
    plt.xscale("symlog")
    plt.barh(pos, data["time"], align='center', height=0.25, color=lightred)    # notice the 'height' argument
    plt.yticks(pos, labels)
    plt.gca().xaxis.set_minor_locator(plt.LogLocator(subs=[0,1,2,3,4,5,6,7,8,9,10]))
    #gca().xaxis.set_minor_formatter(FormatStrFormatter("%.2f"))
    plt.xlabel("time [s]")
    plt.grid(True)

def epsPlot(data, size=(6,3)):
    pos = numpy.arange(len(data))+.5    # the bar centers on the y axis
    labels = list(data["graph"])
    plt.figure(figsize=size)
    plt.xscale("log")
    plt.barh(pos, data["eps"], align='center', height=0.25, color=green)    # notice the 'height' argument
    plt.yticks(pos, labels)
    plt.gca().xaxis.set_minor_locator(plt.LogLocator(subs=[0,1,2,3,4,5,6,7,8,9,10]))
    #gca().xaxis.set_minor_formatter(FormatStrFormatter("%.2f"))
    plt.xlabel("edges / s")
    plt.grid(True)

def graphPlot(data, size=None):
    pos = arange(len(data))    # the bar centers on the y axis
    plt.figure(figsize=(5,3.3))
    sizes = data["m"]
    labels = list(data["graph"])
    #barh(pos, data["n"], log=True, align='center', height=0.4, color="darkgrey")    # notice the 'height' argument
    plt.barh(pos, sizes, log=True, align='center', height=0.4, color="lightblue", label="m")    # notice the 'height' argument
    plt.yticks(pos, labels)
    plt.xlabel("number of edges")
    plt.grid(True)

def barPlot(data, labels, x_label="", y_label="", size=None, color="b", transparency=1.0, scale="linear"):
    pos = numpy.arange(len(data))+.5    # the bar centers on the y axis
    plt.figure(figsize=size)
    plt.xscale(scale)
    plt.barh(pos, data, align='center', height=0.25, color=color, alpha=transparency)    # notice the 'height' argumen
    plt.yticks(pos, labels)
    plt.xlabel(x_label)
    plt.ylabel(y_label)
    plt.grid(True)

class Bench:

    """
    A Bench objects represents a benchmarking session. It is responsible
    for running the benchmarks, collecting and storing the resulting data.

    """

    def __init__(self, graphDir, defaultGraphs, outDir, save=True, nRuns=5, cacheGraphs=False, timeout=None):
        self.defaultGraphs = defaultGraphs
        self.nRuns = nRuns  # default number of runs for each algo
        self.graphDir = graphDir
        # store result data of benchmarks
        self.data = {}
        self.loadTimes = []
        self.outDir = outDir
        self.save = save  # store data frames on disk if true
        # create output directory if it does not exist
        if self.save:
            if not os.path.isdir(self.outDir):
                os.mkdir(self.outDir)
            self.outDataDir = os.path.join(self.outDir, "data")
            self.plotDir = os.path.join(self.outDir, "plots")
            if not os.path.isdir(self.outDataDir):
                os.mkdir(self.outDataDir)
            if not os.path.isdir(self.plotDir):
                os.mkdir(self.plotDir)
            # log file
            self.logPath = os.path.join(self.outDir, "log.txt")
        # graph cache
        self.cacheGraphs = cacheGraphs
        self.graphCache = {}
        #timeout
        self.timeout = timeout

    def getGraph(self, graphName, algo):
        """" Get the graph from disk or from in-memory cache"""
        key = algo.framework + graphName
        if key in self.graphCache:
            self.info("getting {0} from cache".format(graphName))
            G = self.graphCache[key]
            return G
        else:
            self.info("loading {0}".format(graphName))
            with Timer() as t:
                G = algo.loadGraph(os.path.join(self.graphDir, "{0}.gml.graph".format(graphName)))
            self.info("loading graph {0} took {1} s".format(graphName, t.elapsed))
            self.loadTimes.append({"framework" : algo.framework, "graph" : graphName, "time": t.elapsed})
            if self.cacheGraphs:
                self.graphCache[key] = G
            return G

    def getSize(self, G):
        if isinstance(G, networkit.Graph):
            return G.size()
        else:
            raise Error("cannot get graph size - unknown object type")

    def clearCache(self):
        """ Delete all stored graphs to free memory """
        del self.graphCache
        self.graphCache = {}

    def algoBenchmark(self, algo, graphs=None, nRuns=None, timeout=None):
        """ Run a kernel represented by an algorithm benchmark object """
        # set the defaults
        if nRuns is None:
            nRuns = self.nRuns  # lets argument override the default nRuns
        if graphs is None:
            graphs = self.defaultGraphs
        if timeout is None:
            timeout = self.timeout

        self.info("benchmarking {algo.name}".format(**locals()))
        table = []  # list of dictionaries, to be converted to a DataFrame

        for graphName in graphs:
            try:
                G = self.getGraph(graphName, algo)
                (n, m) = self.getSize(G)
                try:
                    self.info("running {algo.name} {nRuns}x on {graphName}".format(**locals()))
                    for i in range(nRuns):
                        row = {}    # benchmark data row
                        row["algo"] = algo.name
                        row["graph"] = graphName
                        row["m"] = m
                        try: # timeout
                            if timeout:
                                signal.signal(signal.SIGALRM, timeoutHandler)
                                signal.alarm(int(timeout * 60))  # timeout in seconds
                            with Timer() as t:
                                result = algo.run(G)
                            self.info("took {0} s".format(t.elapsed))
                            # store data
                            row["time"] = t.elapsed
                            row["eps"] =  m / t.elapsed  # calculate edges per second
                            row["result"] = result
                        except Timeout as tx:
                            self.error("{algo.name} timed out after {timeout} minutes".format(**locals()))
                            row["time"] = None
                            row["eps"] = None
                            row["result"] = None
                        finally:
                            table.append(row)
                            signal.alarm(int(1e9))    # in any case, cancel the timeout alarm by setting it to a ridiculously high time
                except Exception as ex:
                    self.error("algorithm {algo.name} failed with exception: {ex}".format(**locals()))
            except Exception as ex:
                self.error("loading graph {graphName} failed with exception: {ex}".format(**locals()))

        df = pandas.DataFrame(table)
        df.sort("m")    # sort by number of edges
        self.data[algo.name] = df
        # store data frame on disk
        if self.save:
            df.to_csv(os.path.join(self.outDataDir, "{algo.name}.csv".format(**locals())))
        self.log("Done")

    def log(self, message):
        """ Write a message to the logfile"""
        with open(self.logPath, "a") as logfile:
            logfile.write("{0}: {1}\n".format(strftime("%Y-%m-%d %H:%M:%S", gmtime()), message))

    def info(self, message):
        print(message)
        sys.stdout.flush()
        self.log(message)

    def error(self, message):
        print(message)
        sys.stdout.flush()
        self.log(message)

    def debug(self, message):
        pass
        # print(message)

    def timePlot(self, algoName):
        timePlot(averageRuns(self.data[algoName]))
        if self.save:
            plt.savefig(os.path.join(self.plotDir, "{algoName}-time.pdf".format(**locals())), bbox_inches="tight")

    def epsPlot(self, algoName):
        epsPlot(averageRuns(self.data[algoName]))
        if self.save:
            plt.savefig(os.path.join(self.plotDir, "{algoName}-eps.pdf".format(**locals())), bbox_inches="tight")

    def graphPlot(self):
        epsPlot(averageRuns(self.data))
        if self.save:
            plt.savefig(os.path.join(self.plotDir, "graphs.pdf".format(**locals())), bbox_inches="tight")

    def plot(self, algoName):
        print(algoName)
        self.timePlot(algoName)
        self.epsPlot(algoName)

    def plotSummary(self, algoNames=None, figsize=None):
        if algoNames is None:
            algoNames = list(self.data.keys())
        epsSummary = pandas.DataFrame()
        for (algoName, algoData) in self.data.items():
            if algoName in algoNames:
                epsSummary[algoName] = pandas.Series(self.data[algoName]["eps"])

        # data frame
        self.epsSummary = epsSummary
        if self.save:
            epsSummary.to_csv(os.path.join(self.outDataDir, "epsSummary.csv".format(**locals())))
        # plot
        if figsize:
            plt.figure(figsize=figsize)
        plt.gca().xaxis.get_major_formatter().set_powerlimits((3, 3))
        plt.xscale("log")
        plt.xlabel("edges/s")
        seaborn.boxplot(epsSummary, linewidth=1.5, width=.25, color=green, vert=False)
        if self.save:
            plt.savefig(os.path.join(self.plotDir, "epsSummary.pdf".format(**locals())), bbox_inches="tight")

    def plotAll(self):
        for key in self.data.keys():
            self.plot(key)

    def getLoadTimes(self):
        """ Get input times for graphs"""
        return pandas.DataFrame(self.loadTimes)

    def generatorBenchmark(self, generator, argtuples, nRuns=None, timeout=None):
        """ Run a kernel represented by an algorithm benchmark object """
        # set the defaults
        if nRuns is None:
            nRuns = self.nRuns  # lets argument override the default nRuns
        if timeout is None:
            timeout = self.timeout

        genName = str(generator)

        self.info("benchmarking {genName}".format(**locals()))
        table = []  # list of dictionaries, to be converted to a DataFrame

        for param in argtuples:
            try:
                gen = generator(*param)
                row = {}    # benchmark data row
                row["graph"] = str(param)
                try:
                    self.info("running {genName} {nRuns} times".format(**locals()))
                    for i in range(nRuns):
                        row["algo"] = genName
                        try: # timeout
                            result = None
                            if timeout:
                                signal.signal(signal.SIGALRM, timeoutHandler)
                                signal.alarm(int(timeout * 60))  # timeout in seconds
                            with Timer() as t:
                                result = gen.generate()
                            self.debug("took {0} s".format(t.elapsed))
                            # store data
                            row["m"] = result.numberOfNodes()
                            row["time"] = t.elapsed
                            row["result"] = result
                        except Timeout as tx:
                            self.error("{genName} timed out after {timeout} minutes".format(**locals()))
                            row["time"] = None
                            row["result"] = None
                        finally:
                            table.append(row)
                            signal.alarm(int(1e9))    # in any case, cancel the timeout alarm by setting it to a ridiculously high time
                except Exception as ex:
                    self.error("generator {genName} failed with exception: {ex}".format(**locals()))
            except Exception as ex:
                self.error("initializing generator {genName} failed with exception: {ex}".format(**locals()))

        df = pandas.DataFrame(table)
        self.data[genName] = df
        # store data frame on disk
        if self.save:
            df.to_csv(os.path.join(self.outDataDir, "{genName}.csv".format(**locals())))

# - generators
# 	- Erdös-Renyi (generators.ErdosRenyiGenerator)
# 	- Barabasi-Albert (generators.BarabasiAlbertGenerator)
# 	- Chung-Lu (generators.ChungLuGenerator)
# 	- RMAT (generators.RmatGenerator)
<|endoftext|>


<commit_before>"""ClientDistUpdate module: contains the ClientDistUpdate class"""

import DirectNotifyGlobal
import Avatar
import DistributedToon
import Datagram
from MsgTypes import *

class ClientDistUpdate:
    notify = DirectNotifyGlobal.directNotify.newCategory("ClientDistUpdate")

    def __init__(self, dcField):
        self.field = dcField
        self.number = dcField.getNumber()
        self.name = dcField.getName()
        self.types = []
        self.divisors = []
        self.deriveTypesFromParticle(dcField)
        return None

    def deriveTypesFromParticle(self, dcField):
        dcFieldAtomic = dcField.asAtomicField()
        dcFieldMolecular = dcField.asMolecularField()
        if dcFieldAtomic:
            for i in range(0, dcFieldAtomic.getNumElements()):
                self.types.append(dcFieldAtomic.getElementType(i))
                self.divisors.append(dcFieldAtomic.getElementDivisor(i))
        elif dcFieldMolecular:
            for i in range(0, dcFieldMolecular.getNumAtomics()):
                componentField = dcFieldMolecular.getAtomic(i)
                for j in range(0, componentField.getNumElements()):
                    self.types.append(componentField.getElementType(j))
                    self.divisors.append(componentField.getElementDivisor(j))
        else:
            ClientDistUpdate.notify.error("field is neither atom nor molecule")
        return None

    def updateField(self, cdc, do, di):
        # Look up the class
        #aClass = eval(cdc.name + "." + cdc.name)
        # Look up the function
        #assert(aClass.__dict__.has_key(self.name))
        #func = aClass.__dict__[self.name]
        func = eval(cdc.name + "." + cdc.name + "." + self.name)
        # Get the arguments into a list
        args = self.extractArgs(di)
        # Apply the function to the object with the arguments
        apply(func, [do] + args)
        return None

    def extractArgs(self, di):
        args = []
        for i in self.types:
            args.append(di.getArg(i))
        return args

    def addArgs(self, datagram, args):
        # Add the args to the datagram
        numElems = len(args)
        assert (numElems == len(self.types))
        for i in range(0, numElems):
            datagram.putArg(args[i], self.types[i])

    def sendUpdate(self, cr, do, args):
        datagram = Datagram.Datagram()
        # Add message type
        datagram.addUint16(CLIENT_OBJECT_UPDATE_FIELD)
        # Add the DO id
        datagram.addUint32(do.doId)
        # Add the field id
        datagram.addUint16(self.number)
        # Add the arguments
        self.addArgs(datagram, args)
        # send the datagram
        cr.send(datagram)
<commit_msg>*** empty log message ***<commit_after>"""ClientDistUpdate module: contains the ClientDistUpdate class"""

import DirectNotifyGlobal
import Avatar
import DistributedToon
import Datagram
from MsgTypes import *

class ClientDistUpdate:
    notify = DirectNotifyGlobal.directNotify.newCategory("ClientDistUpdate")

    def __init__(self, dcField):
        self.field = dcField
        self.number = dcField.getNumber()
        self.name = dcField.getName()
        self.types = []
        self.divisors = []
        self.deriveTypesFromParticle(dcField)
        return None

    def deriveTypesFromParticle(self, dcField):
        dcFieldAtomic = dcField.asAtomicField()
        dcFieldMolecular = dcField.asMolecularField()
        if dcFieldAtomic:
            for i in range(0, dcFieldAtomic.getNumElements()):
                self.types.append(dcFieldAtomic.getElementType(i))
                self.divisors.append(dcFieldAtomic.getElementDivisor(i))
        elif dcFieldMolecular:
            for i in range(0, dcFieldMolecular.getNumAtomics()):
                componentField = dcFieldMolecular.getAtomic(i)
                for j in range(0, componentField.getNumElements()):
                    self.types.append(componentField.getElementType(j))
                    self.divisors.append(componentField.getElementDivisor(j))
        else:
            ClientDistUpdate.notify.error("field is neither atom nor molecule")
        return None

    def updateField(self, cdc, do, di):
        # Look up the class
        #aClass = eval(cdc.name + "." + cdc.name)
        # Look up the function
        #assert(aClass.__dict__.has_key(self.name))
        #func = aClass.__dict__[self.name]
        func = eval(cdc.name + "." + cdc.name + "." + self.name)
        print("Calling: " + cdc.name + "." + cdc.name + "." + self.name)
        # Get the arguments into a list
        args = self.extractArgs(di)
        # Apply the function to the object with the arguments
        apply(func, [do] + args)
        return None

    def extractArgs(self, di):
        args = []
        for i in self.types:
            args.append(di.getArg(i))
        return args

    def addArgs(self, datagram, args):
        # Add the args to the datagram
        numElems = len(args)
        assert (numElems == len(self.types))
        for i in range(0, numElems):
            datagram.putArg(args[i], self.types[i])

    def sendUpdate(self, cr, do, args):
        datagram = Datagram.Datagram()
        # Add message type
        datagram.addUint16(CLIENT_OBJECT_UPDATE_FIELD)
        # Add the DO id
        datagram.addUint32(do.doId)
        # Add the field id
        datagram.addUint16(self.number)
        # Add the arguments
        self.addArgs(datagram, args)
        # send the datagram
        cr.send(datagram)
<|endoftext|>


<commit_before>from django.db.backends import BaseDatabaseIntrospection
import cx_Oracle
import re

foreign_key_re = re.compile(r"\sCONSTRAINT `[^`]*` FOREIGN KEY \(`([^`]*)`\) REFERENCES `([^`]*)` \(`([^`]*)`\)")

class DatabaseIntrospection(BaseDatabaseIntrospection):
    # Maps type objects to Django Field types.
    data_types_reverse = {
        cx_Oracle.CLOB: 'TextField',
        cx_Oracle.DATETIME: 'DateField',
        cx_Oracle.FIXED_CHAR: 'CharField',
        cx_Oracle.NCLOB: 'TextField',
        cx_Oracle.NUMBER: 'DecimalField',
        cx_Oracle.STRING: 'CharField',
        cx_Oracle.TIMESTAMP: 'DateTimeField',
    }

    try:
        data_types_reverse[cx_Oracle.NATIVE_FLOAT] = 'FloatField'
    except AttributeError:
        pass

    def get_table_list(self, cursor):
        "Returns a list of table names in the current database."
        cursor.execute("SELECT TABLE_NAME FROM USER_TABLES")
        return [row[0].lower() for row in cursor.fetchall()]

    def get_table_description(self, cursor, table_name):
        "Returns a description of the table, with the DB-API cursor.description interface."
        cursor.execute("SELECT * FROM %s WHERE ROWNUM < 2" % self.connection.ops.quote_name(table_name))
        description = []
        for desc in cursor.description:
            description.append((desc[0].lower(),) + desc[1:])
        return description

    def table_name_converter(self, name):
        "Table name comparison is case insensitive under Oracle"
        return name.lower()

    def _name_to_index(self, cursor, table_name):
        """
        Returns a dictionary of {field_name: field_index} for the given table.
        Indexes are 0-based.
        """
        return dict([(d[0], i) for i, d in enumerate(self.get_table_description(cursor, table_name))])

    def get_relations(self, cursor, table_name):
        """
        Returns a dictionary of {field_index: (field_index_other_table, other_table)}
        representing all relationships to the given table. Indexes are 0-based.
        """
        cursor.execute("""
    SELECT ta.column_id - 1, tb.table_name, tb.column_id - 1
    FROM   user_constraints, USER_CONS_COLUMNS ca, USER_CONS_COLUMNS cb,
           user_tab_cols ta, user_tab_cols tb
    WHERE  user_constraints.table_name = %s AND
           ta.table_name = %s AND
           ta.column_name = ca.column_name AND
           ca.table_name = %s AND
           user_constraints.constraint_name = ca.constraint_name AND
           user_constraints.r_constraint_name = cb.constraint_name AND
           cb.table_name = tb.table_name AND
           cb.column_name = tb.column_name AND
           ca.position = cb.position""", [table_name, table_name, table_name])

        relations = {}
        for row in cursor.fetchall():
            relations[row[0]] = (row[2], row[1])
        return relations

    def get_indexes(self, cursor, table_name):
        """
        Returns a dictionary of fieldname -> infodict for the given table,
        where each infodict is in the format:
            {'primary_key': boolean representing whether it's the primary key,
             'unique': boolean representing whether it's a unique index}
        """
        # This query retrieves each index on the given table, including the
        # first associated field name
        # "We were in the nick of time; you were in great peril!"
        sql = """\
SELECT LOWER(all_tab_cols.column_name) AS column_name,
       CASE user_constraints.constraint_type
           WHEN 'P' THEN 1 ELSE 0
       END AS is_primary_key,
       CASE user_indexes.uniqueness
           WHEN 'UNIQUE' THEN 1 ELSE 0
       END AS is_unique
FROM   all_tab_cols, user_cons_columns, user_constraints, user_ind_columns, user_indexes
WHERE  all_tab_cols.column_name = user_cons_columns.column_name (+)
  AND  all_tab_cols.table_name = user_cons_columns.table_name (+)
  AND  user_cons_columns.constraint_name = user_constraints.constraint_name (+)
  AND  user_constraints.constraint_type (+) = 'P'
  AND  user_ind_columns.column_name (+) = all_tab_cols.column_name
  AND  user_ind_columns.table_name (+) = all_tab_cols.table_name
  AND  user_indexes.uniqueness (+) = 'UNIQUE'
  AND  user_indexes.index_name (+) = user_ind_columns.index_name
  AND  all_tab_cols.table_name = UPPER(%s)
"""
        cursor.execute(sql, [table_name])
        indexes = {}
        for row in cursor.fetchall():
            indexes[row[0]] = {'primary_key': row[1], 'unique': row[2]}
        return indexes
<commit_msg>Fixed #11033 -- handle cx_Oracle.UNICODE when driver was compiled without Unicode support.  Thanks, JirkaV.<commit_after>from django.db.backends import BaseDatabaseIntrospection
import cx_Oracle
import re

foreign_key_re = re.compile(r"\sCONSTRAINT `[^`]*` FOREIGN KEY \(`([^`]*)`\) REFERENCES `([^`]*)` \(`([^`]*)`\)")

class DatabaseIntrospection(BaseDatabaseIntrospection):
    # Maps type objects to Django Field types.
    data_types_reverse = {
        cx_Oracle.CLOB: 'TextField',
        cx_Oracle.DATETIME: 'DateField',
        cx_Oracle.FIXED_CHAR: 'CharField',
        cx_Oracle.NCLOB: 'TextField',
        cx_Oracle.NUMBER: 'DecimalField',
        cx_Oracle.STRING: 'CharField',
        cx_Oracle.TIMESTAMP: 'DateTimeField',
    }

    try:
        data_types_reverse[cx_Oracle.NATIVE_FLOAT] = 'FloatField'
    except AttributeError:
        pass

    try:
        data_types_reverse[cx_Oracle.UNICODE] = 'CharField'
    except AttributeError:
        pass

    def get_table_list(self, cursor):
        "Returns a list of table names in the current database."
        cursor.execute("SELECT TABLE_NAME FROM USER_TABLES")
        return [row[0].lower() for row in cursor.fetchall()]

    def get_table_description(self, cursor, table_name):
        "Returns a description of the table, with the DB-API cursor.description interface."
        cursor.execute("SELECT * FROM %s WHERE ROWNUM < 2" % self.connection.ops.quote_name(table_name))
        description = []
        for desc in cursor.description:
            description.append((desc[0].lower(),) + desc[1:])
        return description

    def table_name_converter(self, name):
        "Table name comparison is case insensitive under Oracle"
        return name.lower()

    def _name_to_index(self, cursor, table_name):
        """
        Returns a dictionary of {field_name: field_index} for the given table.
        Indexes are 0-based.
        """
        return dict([(d[0], i) for i, d in enumerate(self.get_table_description(cursor, table_name))])

    def get_relations(self, cursor, table_name):
        """
        Returns a dictionary of {field_index: (field_index_other_table, other_table)}
        representing all relationships to the given table. Indexes are 0-based.
        """
        cursor.execute("""
    SELECT ta.column_id - 1, tb.table_name, tb.column_id - 1
    FROM   user_constraints, USER_CONS_COLUMNS ca, USER_CONS_COLUMNS cb,
           user_tab_cols ta, user_tab_cols tb
    WHERE  user_constraints.table_name = %s AND
           ta.table_name = %s AND
           ta.column_name = ca.column_name AND
           ca.table_name = %s AND
           user_constraints.constraint_name = ca.constraint_name AND
           user_constraints.r_constraint_name = cb.constraint_name AND
           cb.table_name = tb.table_name AND
           cb.column_name = tb.column_name AND
           ca.position = cb.position""", [table_name, table_name, table_name])

        relations = {}
        for row in cursor.fetchall():
            relations[row[0]] = (row[2], row[1])
        return relations

    def get_indexes(self, cursor, table_name):
        """
        Returns a dictionary of fieldname -> infodict for the given table,
        where each infodict is in the format:
            {'primary_key': boolean representing whether it's the primary key,
             'unique': boolean representing whether it's a unique index}
        """
        # This query retrieves each index on the given table, including the
        # first associated field name
        # "We were in the nick of time; you were in great peril!"
        sql = """\
SELECT LOWER(all_tab_cols.column_name) AS column_name,
       CASE user_constraints.constraint_type
           WHEN 'P' THEN 1 ELSE 0
       END AS is_primary_key,
       CASE user_indexes.uniqueness
           WHEN 'UNIQUE' THEN 1 ELSE 0
       END AS is_unique
FROM   all_tab_cols, user_cons_columns, user_constraints, user_ind_columns, user_indexes
WHERE  all_tab_cols.column_name = user_cons_columns.column_name (+)
  AND  all_tab_cols.table_name = user_cons_columns.table_name (+)
  AND  user_cons_columns.constraint_name = user_constraints.constraint_name (+)
  AND  user_constraints.constraint_type (+) = 'P'
  AND  user_ind_columns.column_name (+) = all_tab_cols.column_name
  AND  user_ind_columns.table_name (+) = all_tab_cols.table_name
  AND  user_indexes.uniqueness (+) = 'UNIQUE'
  AND  user_indexes.index_name (+) = user_ind_columns.index_name
  AND  all_tab_cols.table_name = UPPER(%s)
"""
        cursor.execute(sql, [table_name])
        indexes = {}
        for row in cursor.fetchall():
            indexes[row[0]] = {'primary_key': row[1], 'unique': row[2]}
        return indexes
<|endoftext|>