<commit_before>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Paul Nilsson, paul.nilsson@cern.ch, 2017

import os
import time

from pilot.common.errorcodes import ErrorCodes
from pilot.util.container import execute
from pilot.util.constants import SUCCESS, FAILURE

import logging
logger = logging.getLogger(__name__)

errors = ErrorCodes()

def time_stamp():
    """
    Return ISO-8601 compliant date/time format

    :return: time information
    """

    tmptz = time.timezone
    if tmptz > 0:
        signstr = '-'
    else:
        signstr = '+'
    tmptz_hours = int(tmptz / 3600)

    return str("%s%s%02d%02d" % (time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime()), signstr, tmptz_hours,
                                 int(tmptz / 60 - tmptz_hours * 60)))

def get_batchsystem_jobid():
    """
    Identify and return the batch system job id (will be reported to the server)

    :return: batch system job id
    """

    # BQS (e.g. LYON)
    batchsystem_dict = {'QSUB_REQNAME': 'BQS',
                        'BQSCLUSTER': 'BQS',  # BQS alternative
                        'PBS_JOBID': 'Torque',
                        'LSB_JOBID': 'LSF',
                        'JOB_ID': 'Grid Engine',  # Sun's Grid Engine
                        'clusterid': 'Condor',  # Condor (variable sent through job submit file)
                        'SLURM_JOB_ID': 'SLURM'}

    for key, value in batchsystem_dict.iteritems():
        if key in os.environ:
            return value, key

    # Condor (get jobid from classad file)
    if '_CONDOR_JOB_AD' in os.environ:
        from commands import getoutput
        return "Condor", getoutput(
            'sed -n "s/GlobalJobId.*\\"\\(.*\\)\\".*/\\1/p" %s' % os.environ.get("_CONDOR_JOB_AD"))

    return None, ""

def get_job_scheduler_id():
    """
    Get the job scheduler id from the environment variable PANDA_JSID

    :return: job scheduler id (string)
    """
    return os.environ.get("PANDA_JSID", "unknown")

def get_pilot_id():
    """
    Get the pilot id from the environment variable GTAG

    :return: pilot id (string)
    """

    return os.environ.get("GTAG", "unknown")

def whoami():
    """
    Return the name of the pilot user.

    :return: whoami output (string).
    """

    exit_code, who_am_i, stderr = execute('whoami', mute=True)

    return who_am_i

def get_logger(job_id):
    """
    Return the logger object.
    Use this function to get the proper logger object. It relies on a pythno 2.7 function, getChild(), but if the queue
    is only using Python 2.6, the standard logger object will be returned instead.

    :param jod_id: PanDA job id (string).
    :return: logger object.
    """

    try:
        log = logger.getChild(job_id)
    except Exception:
        log = logger
    return log

def shell_exit_code(exit_code):
    """
    Translate the pilot exit code to a proper exit code for the shell (wrapper).

    :param exit_code: pilot error code (int).
    :return: standard shell exit code (int).
    """

    # Error code translation dictionary
    # FORMAT: { pilot_error_code : [ shell_error_code, meaning ], .. }

    # Restricting user (pilot) exit codes to the range 64 - 113, as suggested by http://tldp.org/LDP/abs/html/exitcodes.html
    # Using exit code 137 for kill signal error codes (this actually means a hard kill signal 9, (128+9), 128+2 would mean CTRL+C)

    error_code_translation_dictionary = {
        -1: [64, "Site offline"],
        errors.GENERALERROR: [65, "General pilot error, consult batch log"],
        errors.MKDIR: [66, "Could not create directory"],
        errors.NOSUCHFILE: [67, "No such file or directory"],
        errors.NOVOMSPROXY: [68, "Voms proxy not valid"],
        errors.NOLOCALSPACE: [69, "No space left on local disk"],
        errors.UNKNOWNEXCEPTION: [70, "Exception caught by pilot"],  # same as ERR_PILOTEXC?
        # errors.QUEUEDATA: [71, "Pilot could not download queuedata"],
        # errors.QUEUEDATANOTOK: [72, "Pilot found non-valid queuedata"],
        # errors.NOSOFTWAREDIR: [73, "Software directory does not exist"],
        errors.KILLSIGNAL: [137, "General kill signal"],  # Job terminated by unknown kill signal
        errors.SIGTERM: [143, "Job killed by signal: SIGTERM"],  # 128+15
        errors.SIGQUIT: [131, "Job killed by signal: SIGQUIT"],  # 128+3
        errors.SIGSEGV: [139, "Job killed by signal: SIGSEGV"],  # 128+11
        errors.SIGXCPU: [158, "Job killed by signal: SIGXCPU"],  # 128+30
        errors.SIGUSR1: [144, "Job killed by signal: SIGUSR1"],  # 128+16
        errors.SIGBUS: [138, "Job killed by signal: SIGBUS"]   # 128+10
    }

    if exit_code in error_code_translation_dictionary:
        return error_code_translation_dictionary.get(exit_code)[0]  # Only return the shell exit code, not the error meaning
    elif exit_code != 0:
        logger.warning("no translation to shell exit code for error code %d" % (exit_code))
        return FAILURE
    else:
        return SUCCESS

def declare_failed_by_kill(job, queue, sig):
    """
    Declare the job failed by a kill signal and put it in a suitable failed queue.
    E.g. queue=queues.failed_data_in, if the kill signal was received during stage-in.

    :param job: job object.
    :param queue: queue object.
    :param sig: signal.
    :return:
    """

    job.state = 'failed'
    error_code = errors.get_kill_signal_error_code(sig)
    job.piloterrorcodes, job.piloterrordiags = errors.add_error_code(error_code)

    queue.put(job)

def scan_for_jobs(queues):
    """
    Scan queues until at least one queue has a job object. abort if it takes too long time

    :param queues:
    :return: found jobs (list of job objects).
    """

    t0 = time.time()
    found_job = False
    jobs = None

    while time.time() - t0 < 30:
        for q in queues._fields:
            _q = getattr(queues, q)
            jobs = list(_q.queue)
            if len(jobs) > 0:
                logger.info('found %d job(s) in %d queues after %d s - will begin queue monitoring' %
                            (len(jobs), len(queues._fields), time.time() - t0))
                found_job = True
                break
        if found_job:
            break
        else:
            time.sleep(0.1)

    return jobs

def get_queuedata_from_job(queues):
    """
    Return the queuedata object from a job in the given queues object.
    This function is useful if queuedata is needed from a function that does not know about the job object.
    E.g. the pilot monitor does not know about the job object, but still knows
    about the queues from which a job object can be extracted and therefore the queuedata.

    :param queues: queues object.
    :return: queuedata object.
    """

    # TODO: use scan_for_jobs() instead of the loop below

    job = None
    queuedata = None

    # loop over all queues and find a job object, from which the queuedata object can be reached
    delay = 5
    maxretries = 5
    for q in queues._fields:
        _q = getattr(queues, q)
        jobs = list(_q.queue)
        logger.info('found %d job(s) in %d queues' % (len(jobs), len(queues._fields)))
        if len(jobs) == 0:
            logger.info('taking a 5 s nap')
            time.sleep(5)

        retry = 0
        for job in jobs:
            # allow some time to pass in case no job object has been added to the queues yet
            while retry < maxretries:
                logger.info('attempting to extract queuedata from queues (#%d/#%d)' % (retry + 1, maxretries))
                try:
                    queuedata = job.infosys.queuedata
                except Exception:
                    logger.info('sleeping %d seconds to allow job object to be added to queues '
                                '(needed for queuedata extraction)' % delay)
                    time.sleep(delay)
                    retry += 1
                else:
                    break
        if queuedata:
            break

    return queuedata

def abort_jobs_in_queues(queues, sig):
    """
    Find all jobs in the queues and abort them.

    :param queues: queues object.
    :param sig: detected kill signal.
    :return:
    """

    jobs_list = []

    # loop over all queues and find all jobs
    for q in queues._fields:
        _q = getattr(queues, q)
        jobs = list(_q.queue)
        for job in jobs:
            if job not in jobs_list:
                jobs_list.append(job)

    logger.info('found %d job(s) in %d queues' % (len(jobs_list), len(queues._fields)))
    for job in jobs_list:
        log = get_logger(job.jobid)
        log.info('aborting job %s' % (job.jobid))
        declare_failed_by_kill(job, queues.failed_jobs, sig)
<commit_msg>Added missing put call<commit_after>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Paul Nilsson, paul.nilsson@cern.ch, 2017

import os
import time

from pilot.common.errorcodes import ErrorCodes
from pilot.util.container import execute
from pilot.util.constants import SUCCESS, FAILURE

import logging
logger = logging.getLogger(__name__)

errors = ErrorCodes()

def time_stamp():
    """
    Return ISO-8601 compliant date/time format

    :return: time information
    """

    tmptz = time.timezone
    if tmptz > 0:
        signstr = '-'
    else:
        signstr = '+'
    tmptz_hours = int(tmptz / 3600)

    return str("%s%s%02d%02d" % (time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime()), signstr, tmptz_hours,
                                 int(tmptz / 60 - tmptz_hours * 60)))

def get_batchsystem_jobid():
    """
    Identify and return the batch system job id (will be reported to the server)

    :return: batch system job id
    """

    # BQS (e.g. LYON)
    batchsystem_dict = {'QSUB_REQNAME': 'BQS',
                        'BQSCLUSTER': 'BQS',  # BQS alternative
                        'PBS_JOBID': 'Torque',
                        'LSB_JOBID': 'LSF',
                        'JOB_ID': 'Grid Engine',  # Sun's Grid Engine
                        'clusterid': 'Condor',  # Condor (variable sent through job submit file)
                        'SLURM_JOB_ID': 'SLURM'}

    for key, value in batchsystem_dict.iteritems():
        if key in os.environ:
            return value, key

    # Condor (get jobid from classad file)
    if '_CONDOR_JOB_AD' in os.environ:
        from commands import getoutput
        return "Condor", getoutput(
            'sed -n "s/GlobalJobId.*\\"\\(.*\\)\\".*/\\1/p" %s' % os.environ.get("_CONDOR_JOB_AD"))

    return None, ""

def get_job_scheduler_id():
    """
    Get the job scheduler id from the environment variable PANDA_JSID

    :return: job scheduler id (string)
    """
    return os.environ.get("PANDA_JSID", "unknown")

def get_pilot_id():
    """
    Get the pilot id from the environment variable GTAG

    :return: pilot id (string)
    """

    return os.environ.get("GTAG", "unknown")

def whoami():
    """
    Return the name of the pilot user.

    :return: whoami output (string).
    """

    exit_code, who_am_i, stderr = execute('whoami', mute=True)

    return who_am_i

def get_logger(job_id):
    """
    Return the logger object.
    Use this function to get the proper logger object. It relies on a pythno 2.7 function, getChild(), but if the queue
    is only using Python 2.6, the standard logger object will be returned instead.

    :param jod_id: PanDA job id (string).
    :return: logger object.
    """

    try:
        log = logger.getChild(job_id)
    except Exception:
        log = logger
    return log

def shell_exit_code(exit_code):
    """
    Translate the pilot exit code to a proper exit code for the shell (wrapper).

    :param exit_code: pilot error code (int).
    :return: standard shell exit code (int).
    """

    # Error code translation dictionary
    # FORMAT: { pilot_error_code : [ shell_error_code, meaning ], .. }

    # Restricting user (pilot) exit codes to the range 64 - 113, as suggested by http://tldp.org/LDP/abs/html/exitcodes.html
    # Using exit code 137 for kill signal error codes (this actually means a hard kill signal 9, (128+9), 128+2 would mean CTRL+C)

    error_code_translation_dictionary = {
        -1: [64, "Site offline"],
        errors.GENERALERROR: [65, "General pilot error, consult batch log"],
        errors.MKDIR: [66, "Could not create directory"],
        errors.NOSUCHFILE: [67, "No such file or directory"],
        errors.NOVOMSPROXY: [68, "Voms proxy not valid"],
        errors.NOLOCALSPACE: [69, "No space left on local disk"],
        errors.UNKNOWNEXCEPTION: [70, "Exception caught by pilot"],  # same as ERR_PILOTEXC?
        # errors.QUEUEDATA: [71, "Pilot could not download queuedata"],
        # errors.QUEUEDATANOTOK: [72, "Pilot found non-valid queuedata"],
        # errors.NOSOFTWAREDIR: [73, "Software directory does not exist"],
        errors.KILLSIGNAL: [137, "General kill signal"],  # Job terminated by unknown kill signal
        errors.SIGTERM: [143, "Job killed by signal: SIGTERM"],  # 128+15
        errors.SIGQUIT: [131, "Job killed by signal: SIGQUIT"],  # 128+3
        errors.SIGSEGV: [139, "Job killed by signal: SIGSEGV"],  # 128+11
        errors.SIGXCPU: [158, "Job killed by signal: SIGXCPU"],  # 128+30
        errors.SIGUSR1: [144, "Job killed by signal: SIGUSR1"],  # 128+16
        errors.SIGBUS: [138, "Job killed by signal: SIGBUS"]   # 128+10
    }

    if exit_code in error_code_translation_dictionary:
        return error_code_translation_dictionary.get(exit_code)[0]  # Only return the shell exit code, not the error meaning
    elif exit_code != 0:
        logger.warning("no translation to shell exit code for error code %d" % (exit_code))
        return FAILURE
    else:
        return SUCCESS

def declare_failed_by_kill(job, queue, sig):
    """
    Declare the job failed by a kill signal and put it in a suitable failed queue.
    E.g. queue=queues.failed_data_in, if the kill signal was received during stage-in.

    :param job: job object.
    :param queue: queue object.
    :param sig: signal.
    :return:
    """

    job.state = 'failed'
    error_code = errors.get_kill_signal_error_code(sig)
    job.piloterrorcodes, job.piloterrordiags = errors.add_error_code(error_code)

    queue.put(job)

def scan_for_jobs(queues):
    """
    Scan queues until at least one queue has a job object. abort if it takes too long time

    :param queues:
    :return: found jobs (list of job objects).
    """

    t0 = time.time()
    found_job = False
    jobs = None

    while time.time() - t0 < 30:
        for q in queues._fields:
            _q = getattr(queues, q)
            jobs = list(_q.queue)
            if len(jobs) > 0:
                logger.info('found %d job(s) in queue %s after %d s - will begin queue monitoring' %
                            (len(jobs), q, time.time() - t0))
                found_job = True
                break
        if found_job:
            break
        else:
            time.sleep(0.1)

    return jobs

def get_queuedata_from_job_new(queues):
    """
    Return the queuedata object from a job in the given queues object.
    This function is useful if queuedata is needed from a function that does not know about the job object.
    E.g. the pilot monitor does not know about the job object, but still knows
    about the queues from which a job object can be extracted and therefore the queuedata.

    :param queues: queues object.
    :return: queuedata object.
    """

    # extract jobs from the queues
    jobs = scan_for_jobs(queues)
    if jobs:
        pass

def get_queuedata_from_job(queues):
    """
    Return the queuedata object from a job in the given queues object.
    This function is useful if queuedata is needed from a function that does not know about the job object.
    E.g. the pilot monitor does not know about the job object, but still knows
    about the queues from which a job object can be extracted and therefore the queuedata.

    :param queues: queues object.
    :return: queuedata object.
    """

    # TODO: use scan_for_jobs() instead of the loop below

    job = None
    queuedata = None

    # loop over all queues and find a job object, from which the queuedata object can be reached
    delay = 5
    maxretries = 5
    for q in queues._fields:
        _q = getattr(queues, q)
        jobs = list(_q.queue)
        logger.info('found %d job(s) in queue %s' % (len(jobs), q))
        if len(jobs) == 0:
            logger.info('taking a 5 s nap')
            time.sleep(0.1)

        retry = 0
        for job in jobs:
            # allow some time to pass in case no job object has been added to the queues yet
            while retry < maxretries:
                logger.info('attempting to extract queuedata from queues (#%d/#%d)' % (retry + 1, maxretries))
                try:
                    queuedata = job.infosys.queuedata
                except Exception:
                    logger.info('sleeping %d seconds to allow job object to be added to queues '
                                '(needed for queuedata extraction)' % delay)
                    time.sleep(delay)
                    retry += 1
                else:
                    break
        if queuedata:
            break

    return queuedata

def abort_jobs_in_queues(queues, sig):
    """
    Find all jobs in the queues and abort them.

    :param queues: queues object.
    :param sig: detected kill signal.
    :return:
    """

    jobs_list = []

    # loop over all queues and find all jobs
    for q in queues._fields:
        _q = getattr(queues, q)
        jobs = list(_q.queue)
        for job in jobs:
            if job not in jobs_list:
                jobs_list.append(job)

    logger.info('found %d job(s) in %d queues' % (len(jobs_list), len(queues._fields)))
    for job in jobs_list:
        log = get_logger(job.jobid)
        log.info('aborting job %s' % (job.jobid))
        declare_failed_by_kill(job, queues.failed_jobs, sig)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Paul Nilsson, paul.nilsson@cern.ch, 2017

import os
import time

from pilot.common.errorcodes import ErrorCodes
from pilot.util.container import execute
from pilot.util.constants import SUCCESS, FAILURE

import logging
logger = logging.getLogger(__name__)

errors = ErrorCodes()

def time_stamp():
    """
    Return ISO-8601 compliant date/time format

    :return: time information
    """

    tmptz = time.timezone
    if tmptz > 0:
        signstr = '-'
    else:
        signstr = '+'
    tmptz_hours = int(tmptz / 3600)

    return str("%s%s%02d%02d" % (time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime()), signstr, tmptz_hours,
                                 int(tmptz / 60 - tmptz_hours * 60)))

def get_batchsystem_jobid():
    """
    Identify and return the batch system job id (will be reported to the server)

    :return: batch system job id
    """

    # BQS (e.g. LYON)
    batchsystem_dict = {'QSUB_REQNAME': 'BQS',
                        'BQSCLUSTER': 'BQS',  # BQS alternative
                        'PBS_JOBID': 'Torque',
                        'LSB_JOBID': 'LSF',
                        'JOB_ID': 'Grid Engine',  # Sun's Grid Engine
                        'clusterid': 'Condor',  # Condor (variable sent through job submit file)
                        'SLURM_JOB_ID': 'SLURM'}

    for key, value in batchsystem_dict.iteritems():
        if key in os.environ:
            return value, key

    # Condor (get jobid from classad file)
    if '_CONDOR_JOB_AD' in os.environ:
        from commands import getoutput
        return "Condor", getoutput(
            'sed -n "s/GlobalJobId.*\\"\\(.*\\)\\".*/\\1/p" %s' % os.environ.get("_CONDOR_JOB_AD"))

    return None, ""

def get_job_scheduler_id():
    """
    Get the job scheduler id from the environment variable PANDA_JSID

    :return: job scheduler id (string)
    """
    return os.environ.get("PANDA_JSID", "unknown")

def get_pilot_id():
    """
    Get the pilot id from the environment variable GTAG

    :return: pilot id (string)
    """

    return os.environ.get("GTAG", "unknown")

def whoami():
    """
    Return the name of the pilot user.

    :return: whoami output (string).
    """

    exit_code, who_am_i, stderr = execute('whoami', mute=True)

    return who_am_i

def get_logger(job_id):
    """
    Return the logger object.
    Use this function to get the proper logger object. It relies on a pythno 2.7 function, getChild(), but if the queue
    is only using Python 2.6, the standard logger object will be returned instead.

    :param jod_id: PanDA job id (string).
    :return: logger object.
    """

    try:
        log = logger.getChild(job_id)
    except Exception:
        log = logger
    return log

def shell_exit_code(exit_code):
    """
    Translate the pilot exit code to a proper exit code for the shell (wrapper).

    :param exit_code: pilot error code (int).
    :return: standard shell exit code (int).
    """

    # Error code translation dictionary
    # FORMAT: { pilot_error_code : [ shell_error_code, meaning ], .. }

    # Restricting user (pilot) exit codes to the range 64 - 113, as suggested by http://tldp.org/LDP/abs/html/exitcodes.html
    # Using exit code 137 for kill signal error codes (this actually means a hard kill signal 9, (128+9), 128+2 would mean CTRL+C)

    error_code_translation_dictionary = {
        -1: [64, "Site offline"],
        errors.GENERALERROR: [65, "General pilot error, consult batch log"],
        errors.MKDIR: [66, "Could not create directory"],
        errors.NOSUCHFILE: [67, "No such file or directory"],
        errors.NOVOMSPROXY: [68, "Voms proxy not valid"],
        errors.NOLOCALSPACE: [69, "No space left on local disk"],
        errors.UNKNOWNEXCEPTION: [70, "Exception caught by pilot"],  # same as ERR_PILOTEXC?
        # errors.QUEUEDATA: [71, "Pilot could not download queuedata"],
        # errors.QUEUEDATANOTOK: [72, "Pilot found non-valid queuedata"],
        # errors.NOSOFTWAREDIR: [73, "Software directory does not exist"],
        errors.KILLSIGNAL: [137, "General kill signal"],  # Job terminated by unknown kill signal
        errors.SIGTERM: [143, "Job killed by signal: SIGTERM"],  # 128+15
        errors.SIGQUIT: [131, "Job killed by signal: SIGQUIT"],  # 128+3
        errors.SIGSEGV: [139, "Job killed by signal: SIGSEGV"],  # 128+11
        errors.SIGXCPU: [158, "Job killed by signal: SIGXCPU"],  # 128+30
        errors.SIGUSR1: [144, "Job killed by signal: SIGUSR1"],  # 128+16
        errors.SIGBUS: [138, "Job killed by signal: SIGBUS"]   # 128+10
    }

    if exit_code in error_code_translation_dictionary:
        return error_code_translation_dictionary.get(exit_code)[0]  # Only return the shell exit code, not the error meaning
    elif exit_code != 0:
        logger.warning("no translation to shell exit code for error code %d" % (exit_code))
        return FAILURE
    else:
        return SUCCESS

def declare_failed_by_kill(job, queue, sig):
    """
    Declare the job failed by a kill signal and put it in a suitable failed queue.
    E.g. queue=queues.failed_data_in, if the kill signal was received during stage-in.

    :param job: job object.
    :param queue: queue object.
    :param sig: signal.
    :return:
    """

    job.state = 'failed'
    error_code = errors.get_kill_signal_error_code(sig)
    job.piloterrorcodes, job.piloterrordiags = errors.add_error_code(error_code)

    queue.put(job)

def get_queuedata_from_job(queues):
    """
    Return the queuedata object from a job in the given queues object.
    This function is useful if queuedata is needed from a function that does not know about the job object.
    E.g. the pilot monitor does not know about the job object, but still knows
    about the queues from which a job object can be extracted and therefore the queuedata.

    :param queues: queues object.
    :return: queuedata object.
    """

    job = None
    queuedata = None

    # loop over all queues and find a job object, from which the queuedata object can be reached
    delay = 5
    maxretries = 5
    for q in queues._fields:
        _q = getattr(queues, q)
        jobs = list(_q.queue)
        logger.info('found %d jobs in %d queues' % (len(jobs), len(queues)))
        if len(jobs) == 0:
            logger.info('taking a 5 s nap')
            time.sleep(5)

        retry = 0
        for job in jobs:
            # allow some time to pass in case no job object has been added to the queues yet
            while retry < maxretries:
                logger.info('attempting to extract queuedata from queues (#%d/#%d)' % (retry + 1, maxretries))
                try:
                    queuedata = job.infosys.queuedata
                except Exception:
                    logger.info('sleeping %d seconds to allow job object to be added to queues '
                                '(needed for queuedata extraction)' % delay)
                    time.sleep(delay)
                    retry += 1
                else:
                    break

    return queuedata

def abort_jobs_in_queues(queues, sig):
    """
    Find all jobs in the queues and abort them.

    :param queues: queues object.
    :param sig: detected kill signal.
    :return:
    """

    jobs_list = []

    # loop over all queues and find all jobs
    for q in queues._fields:
        _q = getattr(queues, q)
        jobs = list(_q.queue)
        for job in jobs:
            if job not in jobs_list:
                jobs_list.append(job)

    logger.info('found %d job(s) in %d queues' % (len(jobs_list), len(queues._fields)))
    for job in jobs_list:
        log = get_logger(job.jobid)
        log.info('aborting job %s' % (job.jobid))
        declare_failed_by_kill(job, queues.failed_jobs, sig)
<commit_msg>Corrected usage of len<commit_after>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Paul Nilsson, paul.nilsson@cern.ch, 2017

import os
import time

from pilot.common.errorcodes import ErrorCodes
from pilot.util.container import execute
from pilot.util.constants import SUCCESS, FAILURE

import logging
logger = logging.getLogger(__name__)

errors = ErrorCodes()

def time_stamp():
    """
    Return ISO-8601 compliant date/time format

    :return: time information
    """

    tmptz = time.timezone
    if tmptz > 0:
        signstr = '-'
    else:
        signstr = '+'
    tmptz_hours = int(tmptz / 3600)

    return str("%s%s%02d%02d" % (time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime()), signstr, tmptz_hours,
                                 int(tmptz / 60 - tmptz_hours * 60)))

def get_batchsystem_jobid():
    """
    Identify and return the batch system job id (will be reported to the server)

    :return: batch system job id
    """

    # BQS (e.g. LYON)
    batchsystem_dict = {'QSUB_REQNAME': 'BQS',
                        'BQSCLUSTER': 'BQS',  # BQS alternative
                        'PBS_JOBID': 'Torque',
                        'LSB_JOBID': 'LSF',
                        'JOB_ID': 'Grid Engine',  # Sun's Grid Engine
                        'clusterid': 'Condor',  # Condor (variable sent through job submit file)
                        'SLURM_JOB_ID': 'SLURM'}

    for key, value in batchsystem_dict.iteritems():
        if key in os.environ:
            return value, key

    # Condor (get jobid from classad file)
    if '_CONDOR_JOB_AD' in os.environ:
        from commands import getoutput
        return "Condor", getoutput(
            'sed -n "s/GlobalJobId.*\\"\\(.*\\)\\".*/\\1/p" %s' % os.environ.get("_CONDOR_JOB_AD"))

    return None, ""

def get_job_scheduler_id():
    """
    Get the job scheduler id from the environment variable PANDA_JSID

    :return: job scheduler id (string)
    """
    return os.environ.get("PANDA_JSID", "unknown")

def get_pilot_id():
    """
    Get the pilot id from the environment variable GTAG

    :return: pilot id (string)
    """

    return os.environ.get("GTAG", "unknown")

def whoami():
    """
    Return the name of the pilot user.

    :return: whoami output (string).
    """

    exit_code, who_am_i, stderr = execute('whoami', mute=True)

    return who_am_i

def get_logger(job_id):
    """
    Return the logger object.
    Use this function to get the proper logger object. It relies on a pythno 2.7 function, getChild(), but if the queue
    is only using Python 2.6, the standard logger object will be returned instead.

    :param jod_id: PanDA job id (string).
    :return: logger object.
    """

    try:
        log = logger.getChild(job_id)
    except Exception:
        log = logger
    return log

def shell_exit_code(exit_code):
    """
    Translate the pilot exit code to a proper exit code for the shell (wrapper).

    :param exit_code: pilot error code (int).
    :return: standard shell exit code (int).
    """

    # Error code translation dictionary
    # FORMAT: { pilot_error_code : [ shell_error_code, meaning ], .. }

    # Restricting user (pilot) exit codes to the range 64 - 113, as suggested by http://tldp.org/LDP/abs/html/exitcodes.html
    # Using exit code 137 for kill signal error codes (this actually means a hard kill signal 9, (128+9), 128+2 would mean CTRL+C)

    error_code_translation_dictionary = {
        -1: [64, "Site offline"],
        errors.GENERALERROR: [65, "General pilot error, consult batch log"],
        errors.MKDIR: [66, "Could not create directory"],
        errors.NOSUCHFILE: [67, "No such file or directory"],
        errors.NOVOMSPROXY: [68, "Voms proxy not valid"],
        errors.NOLOCALSPACE: [69, "No space left on local disk"],
        errors.UNKNOWNEXCEPTION: [70, "Exception caught by pilot"],  # same as ERR_PILOTEXC?
        # errors.QUEUEDATA: [71, "Pilot could not download queuedata"],
        # errors.QUEUEDATANOTOK: [72, "Pilot found non-valid queuedata"],
        # errors.NOSOFTWAREDIR: [73, "Software directory does not exist"],
        errors.KILLSIGNAL: [137, "General kill signal"],  # Job terminated by unknown kill signal
        errors.SIGTERM: [143, "Job killed by signal: SIGTERM"],  # 128+15
        errors.SIGQUIT: [131, "Job killed by signal: SIGQUIT"],  # 128+3
        errors.SIGSEGV: [139, "Job killed by signal: SIGSEGV"],  # 128+11
        errors.SIGXCPU: [158, "Job killed by signal: SIGXCPU"],  # 128+30
        errors.SIGUSR1: [144, "Job killed by signal: SIGUSR1"],  # 128+16
        errors.SIGBUS: [138, "Job killed by signal: SIGBUS"]   # 128+10
    }

    if exit_code in error_code_translation_dictionary:
        return error_code_translation_dictionary.get(exit_code)[0]  # Only return the shell exit code, not the error meaning
    elif exit_code != 0:
        logger.warning("no translation to shell exit code for error code %d" % (exit_code))
        return FAILURE
    else:
        return SUCCESS

def declare_failed_by_kill(job, queue, sig):
    """
    Declare the job failed by a kill signal and put it in a suitable failed queue.
    E.g. queue=queues.failed_data_in, if the kill signal was received during stage-in.

    :param job: job object.
    :param queue: queue object.
    :param sig: signal.
    :return:
    """

    job.state = 'failed'
    error_code = errors.get_kill_signal_error_code(sig)
    job.piloterrorcodes, job.piloterrordiags = errors.add_error_code(error_code)

    queue.put(job)

def get_queuedata_from_job(queues):
    """
    Return the queuedata object from a job in the given queues object.
    This function is useful if queuedata is needed from a function that does not know about the job object.
    E.g. the pilot monitor does not know about the job object, but still knows
    about the queues from which a job object can be extracted and therefore the queuedata.

    :param queues: queues object.
    :return: queuedata object.
    """

    job = None
    queuedata = None

    # loop over all queues and find a job object, from which the queuedata object can be reached
    delay = 5
    maxretries = 5
    for q in queues._fields:
        _q = getattr(queues, q)
        jobs = list(_q.queue)
        logger.info('found %d jobs in %d queues' % (len(jobs), len(queues._fields)))
        if len(jobs) == 0:
            logger.info('taking a 5 s nap')
            time.sleep(5)

        retry = 0
        for job in jobs:
            # allow some time to pass in case no job object has been added to the queues yet
            while retry < maxretries:
                logger.info('attempting to extract queuedata from queues (#%d/#%d)' % (retry + 1, maxretries))
                try:
                    queuedata = job.infosys.queuedata
                except Exception:
                    logger.info('sleeping %d seconds to allow job object to be added to queues '
                                '(needed for queuedata extraction)' % delay)
                    time.sleep(delay)
                    retry += 1
                else:
                    break

    return queuedata

def abort_jobs_in_queues(queues, sig):
    """
    Find all jobs in the queues and abort them.

    :param queues: queues object.
    :param sig: detected kill signal.
    :return:
    """

    jobs_list = []

    # loop over all queues and find all jobs
    for q in queues._fields:
        _q = getattr(queues, q)
        jobs = list(_q.queue)
        for job in jobs:
            if job not in jobs_list:
                jobs_list.append(job)

    logger.info('found %d job(s) in %d queues' % (len(jobs_list), len(queues._fields)))
    for job in jobs_list:
        log = get_logger(job.jobid)
        log.info('aborting job %s' % (job.jobid))
        declare_failed_by_kill(job, queues.failed_jobs, sig)
<|endoftext|>


<commit_before>"""Create IAM Instance Profiles, Roles, Users, and Groups."""
import logging

import boto3
from boto3.exceptions import botocore

from .consts import ROLE_POLICY

LOG = logging.getLogger(__name__)

def create_iam_resources(env='dev', group='forrest', app='unicorn'):
    """Create the IAM Resources for the application.

    Args:
        env (str): Deployment environment, i.e. dev, stage, prod.
        group (str): Application Group name.
        app (str): Application name.

    Returns:
        True upon successful completion.
    """
    session = boto3.session.Session(profile_name=env)
    client = session.client('iam')

    user_name = '{group}_{app}'.format(group=group, app=app)
    role_name = '{user}_role'.format(user=user_name)
    profile_name = '{user}_profile'.format(user=user_name)

    resource_action(client,
                    action='create_role',
                    log_format='Role: %(RoleName)s',
                    RoleName=role_name,
                    AssumeRolePolicyDocument=ROLE_POLICY)
    resource_action(client,
                    action='create_instance_profile',
                    log_format='Instance Profile: %(InstanceProfileName)s',
                    InstanceProfileName=profile_name)
    attach_profile_to_role(client,
                           role_name=role_name,
                           profile_name=profile_name)

    resource_action(client,
                    action='create_user',
                    log_format='User: %(UserName)s',
                    UserName=user_name)
    resource_action(client,
                    action='create_group',
                    log_format='Group: %(GroupName)s',
                    GroupName=group)
    resource_action(client,
                    action='add_user_to_group',
                    log_format='User to Group: %(UserName)s -> %(GroupName)s',
                    log_failure=True,
                    GroupName=group,
                    UserName=user_name)

    return True

def attach_profile_to_role(client,
                           role_name='forrest_unicorn_role',
                           profile_name='forrest_unicorn_profile'):
    """Attach an IAM Instance Profile _profile_name_ to Role _role_name_.

    Args:
        role_name (str): Name of Role.
        profile_name (str): Name of Instance Profile.

    Returns:
        True upon successful completion.
    """
    current_instance_profiles = client.list_instance_profiles_for_role(
        RoleName=role_name)['InstanceProfiles']

    for profile in current_instance_profiles:
        if profile['InstanceProfileName'] == profile_name:
            LOG.info('Found Instance Profile attached to Role: %s -> %s',
                     profile_name, role_name)
            break
    else:
        for remove_profile in current_instance_profiles:
            client.remove_role_from_instance_profile(
                InstanceProfileName=remove_profile,
                RoleName=role_name)
            LOG.info('Removed Instance Profile from Role: %s -> %s',
                     remove_profile, role_name)

        client.add_role_to_instance_profile(InstanceProfileName=profile_name,
                                            RoleName=role_name)
        LOG.info('Added Instance Profile to Role: %s -> %s', profile_name,
                 role_name)

    return True

def resource_action(client,
                    action='',
                    log_format='item: %(key)s',
                    log_failure=False,
                    **kwargs):
    """Call _action_ using boto3 _client_ with _kwargs_.

    This is meant for _action_ methods that will create or implicitely prove a
    given Resource exists. The _log_failure_ flag is available for methods that
    should always succeed, but will occasionally fail due to unknown AWS
    issues.

    Args:
        client (botocore.client.IAM): boto3 client object.
        action (str): Client method to call.
        log_format (str): Generic log message format, 'Added' or 'Found' will
            be prepended depending on the scenario.
        log_failure (bool): Will log WARNING level 'Failed' instead of 'Found'
            message.
        **kwargs: Keyword arguments to pass to _action_ method.

    Returns:
        True upon successful completion.
    """
    try:
        getattr(client, action)(**kwargs)
        LOG.info(' '.join(('Added', log_format)), kwargs)
    except botocore.exceptions.ClientError:
        if not log_failure:
            LOG.info(' '.join(('Found', log_format)), kwargs)
        else:
            LOG.warning(' '.join(('Failed', log_format)), kwargs)

    return True
<commit_msg>fix: Remove `_name` for more consistent variables<commit_after>"""Create IAM Instance Profiles, Roles, Users, and Groups."""
import logging

import boto3
from boto3.exceptions import botocore

from .consts import ROLE_POLICY

LOG = logging.getLogger(__name__)

def create_iam_resources(env='dev', group='forrest', app='unicorn'):
    """Create the IAM Resources for the application.

    Args:
        env (str): Deployment environment, i.e. dev, stage, prod.
        group (str): Application Group name.
        app (str): Application name.

    Returns:
        True upon successful completion.
    """
    session = boto3.session.Session(profile_name=env)
    client = session.client('iam')

    user = '{group}_{app}'.format(group=group, app=app)
    role = '{user}_role'.format(user=user)
    profile = '{user}_profile'.format(user=user)

    resource_action(client,
                    action='create_role',
                    log_format='Role: %(RoleName)s',
                    RoleName=role,
                    AssumeRolePolicyDocument=ROLE_POLICY)
    resource_action(client,
                    action='create_instance_profile',
                    log_format='Instance Profile: %(InstanceProfileName)s',
                    InstanceProfileName=profile)
    attach_profile_to_role(client,
                           role_name=role,
                           profile_name=profile)

    resource_action(client,
                    action='create_user',
                    log_format='User: %(UserName)s',
                    UserName=user)
    resource_action(client,
                    action='create_group',
                    log_format='Group: %(GroupName)s',
                    GroupName=group)
    resource_action(client,
                    action='add_user_to_group',
                    log_format='User to Group: %(UserName)s -> %(GroupName)s',
                    log_failure=True,
                    GroupName=group,
                    UserName=user)

    return True

def attach_profile_to_role(client,
                           role_name='forrest_unicorn_role',
                           profile_name='forrest_unicorn_profile'):
    """Attach an IAM Instance Profile _profile_name_ to Role _role_name_.

    Args:
        role_name (str): Name of Role.
        profile_name (str): Name of Instance Profile.

    Returns:
        True upon successful completion.
    """
    current_instance_profiles = client.list_instance_profiles_for_role(
        RoleName=role_name)['InstanceProfiles']

    for profile in current_instance_profiles:
        if profile['InstanceProfileName'] == profile_name:
            LOG.info('Found Instance Profile attached to Role: %s -> %s',
                     profile_name, role_name)
            break
    else:
        for remove_profile in current_instance_profiles:
            client.remove_role_from_instance_profile(
                InstanceProfileName=remove_profile,
                RoleName=role_name)
            LOG.info('Removed Instance Profile from Role: %s -> %s',
                     remove_profile, role_name)

        client.add_role_to_instance_profile(InstanceProfileName=profile_name,
                                            RoleName=role_name)
        LOG.info('Added Instance Profile to Role: %s -> %s', profile_name,
                 role_name)

    return True

def resource_action(client,
                    action='',
                    log_format='item: %(key)s',
                    log_failure=False,
                    **kwargs):
    """Call _action_ using boto3 _client_ with _kwargs_.

    This is meant for _action_ methods that will create or implicitely prove a
    given Resource exists. The _log_failure_ flag is available for methods that
    should always succeed, but will occasionally fail due to unknown AWS
    issues.

    Args:
        client (botocore.client.IAM): boto3 client object.
        action (str): Client method to call.
        log_format (str): Generic log message format, 'Added' or 'Found' will
            be prepended depending on the scenario.
        log_failure (bool): Will log WARNING level 'Failed' instead of 'Found'
            message.
        **kwargs: Keyword arguments to pass to _action_ method.

    Returns:
        True upon successful completion.
    """
    try:
        getattr(client, action)(**kwargs)
        LOG.info(' '.join(('Added', log_format)), kwargs)
    except botocore.exceptions.ClientError:
        if not log_failure:
            LOG.info(' '.join(('Found', log_format)), kwargs)
        else:
            LOG.warning(' '.join(('Failed', log_format)), kwargs)

    return True
<|endoftext|>


<commit_before>################################### LICENSE ####################################
#                            Copyright 2016 Morphux                            #
#                                                                              #
#        Licensed under the Apache License, Version 2.0 (the "License");       #
#        you may not use this file except in compliance with the License.      #
#                  You may obtain a copy of the License at                     #
#                                                                              #
#                 http://www.apache.org/licenses/LICENSE-2.0                   #
#                                                                              #
#      Unless required by applicable law or agreed to in writing, software     #
#       distributed under the License is distributed on an "AS IS" BASIS,      #
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  #
#        See the License for the specific language governing permissions and   #
#                       limitations under the License.                         #
################################################################################

##
# gdbm_p2.py
# Created: 21/12/2016
# By: Louis Solofrizzo <louis@morphux.org>
##

import      os

class   Gdbm_P2:

    conf_lst = {}
    e = False
    root_dir = ""

    def     init(self, c_lst, ex, root_dir):
        self.conf_lst = c_lst
        self.e = ex
        self.root_dir = root_dir
        self.config = {
            "name": "gdbm", # Name of the package
            "version": "1.12", # Version of the package
            "size": 8.8, # Size of the installed package (MB)
            "archive": "", # Archive name
            "SBU": 0.1, # SBU (Compilation time)
            "tmp_install": False, # Is this package part of the temporary install
            "next": "gperf", # Next package to install
            "before": False,
            "after": False,
            "urls": [ # Url to download the package. The first one must be morphux servers
                "https://install.morphux.org/packages/"
            ]
        }
        return self.config

    def     configure(self):
        return self.e(["./configure",
                "--prefix=/usr",
                "--disable-static",
                "--enable-libgdbm-compat"
            ])

    def     make(self):
        return self.e(["make", "-j", self.conf_lst["cpus"]])

    def     install(self):
        return self.e(["make", "install"])
<commit_msg>Fix(gdbm_p2 package): Add archive url<commit_after>################################### LICENSE ####################################
#                            Copyright 2016 Morphux                            #
#                                                                              #
#        Licensed under the Apache License, Version 2.0 (the "License");       #
#        you may not use this file except in compliance with the License.      #
#                  You may obtain a copy of the License at                     #
#                                                                              #
#                 http://www.apache.org/licenses/LICENSE-2.0                   #
#                                                                              #
#      Unless required by applicable law or agreed to in writing, software     #
#       distributed under the License is distributed on an "AS IS" BASIS,      #
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  #
#        See the License for the specific language governing permissions and   #
#                       limitations under the License.                         #
################################################################################

##
# gdbm_p2.py
# Created: 21/12/2016
# By: Louis Solofrizzo <louis@morphux.org>
##

import      os

class   Gdbm_P2:

    conf_lst = {}
    e = False
    root_dir = ""

    def     init(self, c_lst, ex, root_dir):
        self.conf_lst = c_lst
        self.e = ex
        self.root_dir = root_dir
        self.config = {
            "name": "gdbm", # Name of the package
            "version": "1.12", # Version of the package
            "size": 8.8, # Size of the installed package (MB)
            "archive": "gdbm-1.12.tar.gz", # Archive name
            "SBU": 0.1, # SBU (Compilation time)
            "tmp_install": False, # Is this package part of the temporary install
            "next": "gperf", # Next package to install
            "before": False,
            "after": False,
            "urls": [ # Url to download the package. The first one must be morphux servers
                "https://install.morphux.org/packages/gdbm-1.12.tar.gz"
            ]
        }
        return self.config

    def     configure(self):
        return self.e(["./configure",
                "--prefix=/usr",
                "--disable-static",
                "--enable-libgdbm-compat"
            ])

    def     make(self):
        return self.e(["make", "-j", self.conf_lst["cpus"]])

    def     install(self):
        return self.e(["make", "install"])
<|endoftext|>


<commit_before>from datetime import date, datetime

from django.db import models
from django.db.models import Q

from django.utils.translation import ugettext_lazy as _

import plata
from plata.compat import product as itertools_product
from plata.fields import CurrencyField

class TaxClass(models.Model):
    name = models.CharField(_('name'), max_length=100)
    rate = models.DecimalField(_('rate'), max_digits=10, decimal_places=2)
    priority = models.PositiveIntegerField(_('priority'), default=0,
        help_text = _('Used to order the tax classes in the administration interface.'))

    class Meta:
        ordering = ['-priority']
        verbose_name = _('tax class')
        verbose_name_plural = _('tax classes')

    def __unicode__(self):
        return self.name

class CategoryManager(models.Manager):
    def active(self):
        return self.filter(is_active=True)

    def public(self):
        return self.filter(is_active=True, is_internal=False)

class Category(models.Model):
    is_active = models.BooleanField(_('is active'), default=True)
    is_internal = models.BooleanField(_('is internal'), default=False,
        help_text=_('Only used to internally organize products, f.e. for discounting.'))

    name = models.CharField(_('name'), max_length=100)
    slug = models.SlugField(_('slug'), unique=True)
    ordering = models.PositiveIntegerField(_('ordering'), default=0)
    description = models.TextField(_('description'), blank=True)

    parent = models.ForeignKey('self', blank=True, null=True,
        limit_choices_to={'parent__isnull': True},
        related_name='children', verbose_name=_('parent'))

    class Meta:
        ordering = ['parent__ordering', 'parent__name', 'ordering', 'name']
        verbose_name = _('category')
        verbose_name_plural = _('categories')

    objects = CategoryManager()

    def __unicode__(self):
        if self.parent_id:
            return u'%s - %s' % (self.parent, self.name)
        return self.name

    @models.permalink
    def get_absolute_url(self):
        return ('plata_category_detail', (), {'object_id': self.pk})

class OptionGroup(models.Model):
    name = models.CharField(_('name'), max_length=100)

    class Meta:
        verbose_name = _('option group')
        verbose_name_plural = _('option groups')

    def __unicode__(self):
        return self.name

class Option(models.Model):
    group = models.ForeignKey(OptionGroup, related_name='options',
        verbose_name=_('option group'))
    name = models.CharField(_('name'), max_length=100)
    value = models.CharField(_('value'), max_length=100)
    ordering = models.PositiveIntegerField(_('ordering'), default=0)

    class Meta:
        ordering = ['group', 'ordering']
        verbose_name = _('option')
        verbose_name_plural = _('options')

    def __unicode__(self):
        return self.name

    def full_name(self):
        return u'%s - %s' % (self.group, self.name)

class ProductManager(models.Manager):
    def active(self):
        return self.filter(is_active=True)

class Product(models.Model):
    is_active = models.BooleanField(_('is active'), default=True)
    is_featured = models.BooleanField(_('is featured'), default=False)
    name = models.CharField(_('name'), max_length=100)
    slug = models.SlugField(_('slug'), unique=True)
    ordering = models.PositiveIntegerField(_('ordering'), default=0)
    sku = models.CharField(_('SKU'), max_length=100, unique=True)

    categories = models.ManyToManyField(Category,
        verbose_name=_('categories'), related_name='products',
        blank=True, null=True)
    description = models.TextField(_('description'), blank=True)
    option_groups = models.ManyToManyField(OptionGroup, related_name='products',
        blank=True, null=True, verbose_name=_('option groups'))

    class Meta:
        ordering = ['ordering', 'name']
        verbose_name = _('product')
        verbose_name_plural = _('products')

    objects = ProductManager()

    def __unicode__(self):
        return self.name

    def save(self, *args, **kwargs):
        if not self.sku:
            self.sku = self.slug
        super(Product, self).save(*args, **kwargs)

    @models.permalink
    def get_absolute_url(self):
        return ('plata_product_detail', (), {'object_id': self.pk})

    @property
    def main_image(self):
        if not hasattr(self, '_main_image'):
            try:
                self._main_image = self.images.all()[0]
            except IndexError:
                self._main_image = None
        return self._main_image

    def get_price(self, **kwargs):
        return self.prices.active().filter(**kwargs).latest()

    def get_prices(self):
        from django.core.cache import cache

        key = 'product-prices-%s' % self.pk

        if cache.has_key(key):
            return cache.get(key)

        # TODO determine currencies differently
        currencies = ('CHF', 'EUR', 'USD')

        prices = []
        for currency in currencies:
            try:
                normal, sale = self.prices.active().filter(currency=currency).latest(), None
            except self.prices.model.DoesNotExist:
                continue

            if normal.is_sale:
                sale = normal
                try:
                    normal = self.prices.active().filter(is_sale=False, currency=currency).latest()
                except self.prices.model.DoesNotExist:
                    normal = None

            prices.append((currency, {
                'normal': normal,
                'sale': sale,
                }))

        cache.set(key, prices)
        return prices

    def in_sale(self, currency):
        prices = self.get_prices()
        if 'currency' in prices and prices[currency]['sale']:
            return True
        return False

    def create_variations(self):
        variations = itertools_product(*[group.options.all() for group in self.option_groups.all()])

        for idx, variation in enumerate(variations):
            try:
                qset = self.variations
                for o in variation:
                    qset = qset.filter(options=o)

                instance = qset.get()
            except ProductVariation.DoesNotExist:
                parts = [self.sku]
                parts.extend(o.value for o in variation)
                instance = self.variations.create(
                    is_active=self.is_active,
                    sku=u'-'.join(parts),
                    )
                instance.options = variation
            except ProductVariation.MultipleObjectsReturned:
                raise Exception('DAMN!')

            instance.ordering = idx
            instance.save()

class ProductVariation(models.Model):
    product = models.ForeignKey(Product, related_name='variations')
    is_active = models.BooleanField(_('is active'), default=True)
    sku = models.CharField(_('SKU'), max_length=100, unique=True)
    items_in_stock = models.IntegerField(_('items in stock'), default=0)
    options = models.ManyToManyField(Option, related_name='products',
        blank=True, null=True, verbose_name=_('options'))
    ordering = models.PositiveIntegerField(_('ordering'), default=0)

    class Meta:
        ordering = ['ordering', 'product']
        verbose_name = _('product variation')
        verbose_name_plural = _('product variations')

    def __unicode__(self):
        options = u', '.join(unicode(o) for o in self.options.all())

        if options:
            return u'%s (%s)' % (self.product, options)

        return u'%s' % self.product

    def get_absolute_url(self):
        return self.product.get_absolute_url()

class ProductPriceManager(models.Manager):
    def active(self):
        return self.filter(
            Q(is_active=True),
            Q(valid_from__lte=date.today()),
            Q(valid_until__isnull=True) | Q(valid_until__gte=date.today()))

class ProductPrice(models.Model):
    product = models.ForeignKey(Product, verbose_name=_('product'),
        related_name='prices')
    currency = CurrencyField()
    _unit_price = models.DecimalField(_('unit price'), max_digits=18, decimal_places=10)
    tax_included = models.BooleanField(_('tax included'),
        help_text=_('Is tax included in given unit price?'),
        default=plata.settings.PLATA_PRICE_INCLUDES_TAX)
    tax_class = models.ForeignKey(TaxClass, verbose_name=_('tax class'))

    is_active = models.BooleanField(_('is active'), default=True)
    valid_from = models.DateField(_('valid from'), default=date.today)
    valid_until = models.DateField(_('valid until'), blank=True, null=True)

    is_sale = models.BooleanField(_('is sale'), default=False,
        help_text=_('Set this if this price is a sale price. Whether the sale is temporary or not does not matter.'))

    class Meta:
        get_latest_by = 'id'
        ordering = ['-valid_from']
        verbose_name = _('product price')
        verbose_name_plural = _('product prices')

    objects = ProductPriceManager()

    def __unicode__(self):
        return u'%s %.2f' % (self.currency, self.unit_price)

    @property
    def unit_tax(self):
        return self.unit_price_excl_tax * (self.tax_class.rate/100)

    @property
    def unit_price_incl_tax(self):
        if self.tax_included:
            return self._unit_price
        return self._unit_price * (1+self.tax_class.rate/100)

    @property
    def unit_price_excl_tax(self):
        if not self.tax_included:
            return self._unit_price
        return self._unit_price / (1+self.tax_class.rate/100)

    @property
    def unit_price(self):
        if plata.settings.PLATA_PRICE_INCLUDES_TAX:
            return self.unit_price_incl_tax
        else:
            return self.unit_price_excl_tax

class ProductImage(models.Model):
    product = models.ForeignKey(Product, verbose_name=_('product'),
        related_name='images')
    image = models.ImageField(_('image'),
        upload_to=lambda instance, filename: 'products/%s/%s' % (instance.product.slug, filename))
    ordering = models.PositiveIntegerField(_('ordering'), default=0)

    class Meta:
        ordering = ['ordering']
        verbose_name = _('product image')
        verbose_name_plural = _('product images')

    def __unicode__(self):
        return self.image.name
<commit_msg>Fix in_sale helper<commit_after>from datetime import date, datetime

from django.db import models
from django.db.models import Q

from django.utils.translation import ugettext_lazy as _

import plata
from plata.compat import product as itertools_product
from plata.fields import CurrencyField

class TaxClass(models.Model):
    name = models.CharField(_('name'), max_length=100)
    rate = models.DecimalField(_('rate'), max_digits=10, decimal_places=2)
    priority = models.PositiveIntegerField(_('priority'), default=0,
        help_text = _('Used to order the tax classes in the administration interface.'))

    class Meta:
        ordering = ['-priority']
        verbose_name = _('tax class')
        verbose_name_plural = _('tax classes')

    def __unicode__(self):
        return self.name

class CategoryManager(models.Manager):
    def active(self):
        return self.filter(is_active=True)

    def public(self):
        return self.filter(is_active=True, is_internal=False)

class Category(models.Model):
    is_active = models.BooleanField(_('is active'), default=True)
    is_internal = models.BooleanField(_('is internal'), default=False,
        help_text=_('Only used to internally organize products, f.e. for discounting.'))

    name = models.CharField(_('name'), max_length=100)
    slug = models.SlugField(_('slug'), unique=True)
    ordering = models.PositiveIntegerField(_('ordering'), default=0)
    description = models.TextField(_('description'), blank=True)

    parent = models.ForeignKey('self', blank=True, null=True,
        limit_choices_to={'parent__isnull': True},
        related_name='children', verbose_name=_('parent'))

    class Meta:
        ordering = ['parent__ordering', 'parent__name', 'ordering', 'name']
        verbose_name = _('category')
        verbose_name_plural = _('categories')

    objects = CategoryManager()

    def __unicode__(self):
        if self.parent_id:
            return u'%s - %s' % (self.parent, self.name)
        return self.name

    @models.permalink
    def get_absolute_url(self):
        return ('plata_category_detail', (), {'object_id': self.pk})

class OptionGroup(models.Model):
    name = models.CharField(_('name'), max_length=100)

    class Meta:
        verbose_name = _('option group')
        verbose_name_plural = _('option groups')

    def __unicode__(self):
        return self.name

class Option(models.Model):
    group = models.ForeignKey(OptionGroup, related_name='options',
        verbose_name=_('option group'))
    name = models.CharField(_('name'), max_length=100)
    value = models.CharField(_('value'), max_length=100)
    ordering = models.PositiveIntegerField(_('ordering'), default=0)

    class Meta:
        ordering = ['group', 'ordering']
        verbose_name = _('option')
        verbose_name_plural = _('options')

    def __unicode__(self):
        return self.name

    def full_name(self):
        return u'%s - %s' % (self.group, self.name)

class ProductManager(models.Manager):
    def active(self):
        return self.filter(is_active=True)

class Product(models.Model):
    is_active = models.BooleanField(_('is active'), default=True)
    is_featured = models.BooleanField(_('is featured'), default=False)
    name = models.CharField(_('name'), max_length=100)
    slug = models.SlugField(_('slug'), unique=True)
    ordering = models.PositiveIntegerField(_('ordering'), default=0)
    sku = models.CharField(_('SKU'), max_length=100, unique=True)

    categories = models.ManyToManyField(Category,
        verbose_name=_('categories'), related_name='products',
        blank=True, null=True)
    description = models.TextField(_('description'), blank=True)
    option_groups = models.ManyToManyField(OptionGroup, related_name='products',
        blank=True, null=True, verbose_name=_('option groups'))

    class Meta:
        ordering = ['ordering', 'name']
        verbose_name = _('product')
        verbose_name_plural = _('products')

    objects = ProductManager()

    def __unicode__(self):
        return self.name

    def save(self, *args, **kwargs):
        if not self.sku:
            self.sku = self.slug
        super(Product, self).save(*args, **kwargs)

    @models.permalink
    def get_absolute_url(self):
        return ('plata_product_detail', (), {'object_id': self.pk})

    @property
    def main_image(self):
        if not hasattr(self, '_main_image'):
            try:
                self._main_image = self.images.all()[0]
            except IndexError:
                self._main_image = None
        return self._main_image

    def get_price(self, **kwargs):
        return self.prices.active().filter(**kwargs).latest()

    def get_prices(self):
        from django.core.cache import cache

        key = 'product-prices-%s' % self.pk

        if cache.has_key(key):
            return cache.get(key)

        # TODO determine currencies differently
        currencies = ('CHF', 'EUR', 'USD')

        prices = []
        for currency in currencies:
            try:
                normal, sale = self.prices.active().filter(currency=currency).latest(), None
            except self.prices.model.DoesNotExist:
                continue

            if normal.is_sale:
                sale = normal
                try:
                    normal = self.prices.active().filter(is_sale=False, currency=currency).latest()
                except self.prices.model.DoesNotExist:
                    normal = None

            prices.append((currency, {
                'normal': normal,
                'sale': sale,
                }))

        cache.set(key, prices)
        return prices

    def in_sale(self, currency):
        prices = dict(self.get_prices())
        if currency in prices and prices[currency]['sale']:
            return True
        return False

    def create_variations(self):
        variations = itertools_product(*[group.options.all() for group in self.option_groups.all()])

        for idx, variation in enumerate(variations):
            try:
                qset = self.variations
                for o in variation:
                    qset = qset.filter(options=o)

                instance = qset.get()
            except ProductVariation.DoesNotExist:
                parts = [self.sku]
                parts.extend(o.value for o in variation)
                instance = self.variations.create(
                    is_active=self.is_active,
                    sku=u'-'.join(parts),
                    )
                instance.options = variation
            except ProductVariation.MultipleObjectsReturned:
                raise Exception('DAMN!')

            instance.ordering = idx
            instance.save()

class ProductVariation(models.Model):
    product = models.ForeignKey(Product, related_name='variations')
    is_active = models.BooleanField(_('is active'), default=True)
    sku = models.CharField(_('SKU'), max_length=100, unique=True)
    items_in_stock = models.IntegerField(_('items in stock'), default=0)
    options = models.ManyToManyField(Option, related_name='products',
        blank=True, null=True, verbose_name=_('options'))
    ordering = models.PositiveIntegerField(_('ordering'), default=0)

    class Meta:
        ordering = ['ordering', 'product']
        verbose_name = _('product variation')
        verbose_name_plural = _('product variations')

    def __unicode__(self):
        options = u', '.join(unicode(o) for o in self.options.all())

        if options:
            return u'%s (%s)' % (self.product, options)

        return u'%s' % self.product

    def get_absolute_url(self):
        return self.product.get_absolute_url()

class ProductPriceManager(models.Manager):
    def active(self):
        return self.filter(
            Q(is_active=True),
            Q(valid_from__lte=date.today()),
            Q(valid_until__isnull=True) | Q(valid_until__gte=date.today()))

class ProductPrice(models.Model):
    product = models.ForeignKey(Product, verbose_name=_('product'),
        related_name='prices')
    currency = CurrencyField()
    _unit_price = models.DecimalField(_('unit price'), max_digits=18, decimal_places=10)
    tax_included = models.BooleanField(_('tax included'),
        help_text=_('Is tax included in given unit price?'),
        default=plata.settings.PLATA_PRICE_INCLUDES_TAX)
    tax_class = models.ForeignKey(TaxClass, verbose_name=_('tax class'))

    is_active = models.BooleanField(_('is active'), default=True)
    valid_from = models.DateField(_('valid from'), default=date.today)
    valid_until = models.DateField(_('valid until'), blank=True, null=True)

    is_sale = models.BooleanField(_('is sale'), default=False,
        help_text=_('Set this if this price is a sale price. Whether the sale is temporary or not does not matter.'))

    class Meta:
        get_latest_by = 'id'
        ordering = ['-valid_from']
        verbose_name = _('product price')
        verbose_name_plural = _('product prices')

    objects = ProductPriceManager()

    def __unicode__(self):
        return u'%s %.2f' % (self.currency, self.unit_price)

    @property
    def unit_tax(self):
        return self.unit_price_excl_tax * (self.tax_class.rate/100)

    @property
    def unit_price_incl_tax(self):
        if self.tax_included:
            return self._unit_price
        return self._unit_price * (1+self.tax_class.rate/100)

    @property
    def unit_price_excl_tax(self):
        if not self.tax_included:
            return self._unit_price
        return self._unit_price / (1+self.tax_class.rate/100)

    @property
    def unit_price(self):
        if plata.settings.PLATA_PRICE_INCLUDES_TAX:
            return self.unit_price_incl_tax
        else:
            return self.unit_price_excl_tax

class ProductImage(models.Model):
    product = models.ForeignKey(Product, verbose_name=_('product'),
        related_name='images')
    image = models.ImageField(_('image'),
        upload_to=lambda instance, filename: 'products/%s/%s' % (instance.product.slug, filename))
    ordering = models.PositiveIntegerField(_('ordering'), default=0)

    class Meta:
        ordering = ['ordering']
        verbose_name = _('product image')
        verbose_name_plural = _('product images')

    def __unicode__(self):
        return self.image.name
<|endoftext|>


<commit_before>import sublime

PLUGIN_NAME = 'LSP'

try:
    from typing import List, Optional, Dict
    assert List and Optional and Dict
except ImportError:
    pass

def read_bool_setting(settings_obj: sublime.Settings, key: str, default: bool) -> bool:
    val = settings_obj.get(key)
    if isinstance(val, bool):
        return val
    else:
        return default

def read_dict_setting(settings_obj: sublime.Settings, key: str, default: dict) -> dict:
    val = settings_obj.get(key)
    if isinstance(val, dict):
        return val
    else:
        return default

def read_str_setting(settings_obj: sublime.Settings, key: str, default: str) -> str:
    val = settings_obj.get(key)
    if isinstance(val, str):
        return val
    else:
        return default

class Settings(object):

    def __init__(self):
        self.show_status_messages = True
        self.show_view_status = True
        self.auto_show_diagnostics_panel = True
        self.show_diagnostics_phantoms = False
        self.show_diagnostics_in_view_status = True
        self.only_show_lsp_completions = False
        self.diagnostics_highlight_style = "underline"
        self.document_highlight_style = "stippled"
        self.document_highlight_scopes = {
            "unknown": "text",
            "text": "text",
            "read": "markup.inserted",
            "write": "markup.changed"
        }
        self.diagnostics_gutter_marker = "dot"
        self.complete_all_chars = False
        self.completion_hint_type = "auto"
        self.resolve_completion_for_snippets = False
        self.log_debug = True
        self.log_server = True
        self.log_stderr = False
        self.log_payloads = False

    def update(self, settings_obj: sublime.Settings):
        self.show_status_messages = read_bool_setting(settings_obj, "show_status_messages", True)
        self.show_view_status = read_bool_setting(settings_obj, "show_view_status", True)
        self.auto_show_diagnostics_panel = read_bool_setting(settings_obj, "auto_show_diagnostics_panel", True)
        self.show_diagnostics_phantoms = read_bool_setting(settings_obj, "show_diagnostics_phantoms", False)
        self.show_diagnostics_in_view_status = read_bool_setting(settings_obj, "show_diagnostics_in_view_status", True)
        self.diagnostics_highlight_style = read_str_setting(settings_obj, "diagnostics_highlight_style", "underline")
        self.document_highlight_style = read_str_setting(settings_obj, "document_highlight_style", "stippled")
        self.document_highlight_scopes = read_dict_setting(
            settings_obj, "document_highlight_scopes", {
                "unknown": "text",
                "text": "text",
                "read": "markup.inserted",
                "write": "markup.changed"
            })
        self.diagnostics_gutter_marker = read_str_setting(settings_obj, "diagnostics_gutter_marker", "dot")
        self.only_show_lsp_completions = read_bool_setting(settings_obj, "only_show_lsp_completions", False)
        self.complete_all_chars = read_bool_setting(settings_obj, "complete_all_chars", True)
        self.completion_hint_type = read_str_setting(settings_obj, "completion_hint_type", "auto")
        self.resolve_completion_for_snippets = read_bool_setting(settings_obj, "resolve_completion_for_snippets", False)
        self.log_debug = read_bool_setting(settings_obj, "log_debug", False)
        self.log_server = read_bool_setting(settings_obj, "log_server", True)
        self.log_stderr = read_bool_setting(settings_obj, "log_stderr", False)
        self.log_payloads = read_bool_setting(settings_obj, "log_payloads", False)

class ClientConfigs(object):

    def __init__(self):
        self._default_settings = dict()  # type: Dict[str, dict]
        self._global_settings = dict()  # type: Dict[str, dict]
        self.defaults = []  # type: List[ClientConfig]
        self.all = []  # type: List[ClientConfig]

    def update(self, settings_obj: sublime.Settings):
        self._default_settings = read_dict_setting(settings_obj, "default_clients", {})
        self._global_settings = read_dict_setting(settings_obj, "clients", {})

        self.defaults = read_client_configs(self._default_settings)
        self.all = read_client_configs(self._global_settings, self._default_settings)

        client_enableds = list("=".join([config.name, str(config.enabled)]) for config in self.all)
        print(PLUGIN_NAME + ': global clients: ' + ", ".join(client_enableds))

    def _set_enabled(self, config_name: str, is_enabled: bool):
        if _settings_obj:
            client_settings = self._global_settings.setdefault(config_name, {})
            client_settings["enabled"] = is_enabled
            _settings_obj.set("clients", self._global_settings)
            sublime.save_settings("LSP.sublime-settings")

    def enable(self, config_name: str):
        self._set_enabled(config_name, True)

    def disable(self, config_name: str):
        self._set_enabled(config_name, False)

_settings_obj = None  # type: Optional[sublime.Settings]
settings = Settings()
client_configs = ClientConfigs()

def load_settings():
    global _settings_obj
    loaded_settings_obj = sublime.load_settings("LSP.sublime-settings")
    _settings_obj = loaded_settings_obj
    settings.update(loaded_settings_obj)
    client_configs.update(loaded_settings_obj)
    loaded_settings_obj.add_on_change("_on_new_settings", lambda: settings.update(loaded_settings_obj))
    loaded_settings_obj.add_on_change("_on_new_client_settings", lambda: client_configs.update(loaded_settings_obj))

def unload_settings():
    if _settings_obj:
        _settings_obj.clear_on_change("_on_new_settings")
        _settings_obj.clear_on_change("_on_new_client_settings")

class ClientConfig(object):
    def __init__(self, name, binary_args, scopes, syntaxes, languageId,
                 enabled=True, init_options=dict(), settings=dict(), env=dict()):
        self.name = name
        self.binary_args = binary_args
        self.scopes = scopes
        self.syntaxes = syntaxes
        self.languageId = languageId
        self.enabled = enabled
        self.init_options = init_options
        self.settings = settings
        self.env = env

def read_client_config(name, client_config):
    return ClientConfig(
        name,
        client_config.get("command", []),
        client_config.get("scopes", []),
        client_config.get("syntaxes", []),
        client_config.get("languageId", ""),
        client_config.get("enabled", True),
        client_config.get("initializationOptions", dict()),
        client_config.get("settings", dict()),
        client_config.get("env", dict())
    )

def read_client_configs(client_settings, default_client_settings=None) -> 'List[ClientConfig]':
    parsed_configs = []  # type: List[ClientConfig]
    if isinstance(client_settings, dict):
        for client_name, client_config in client_settings.items():

            # start with default settings for this client if available
            client_with_defaults = {}  # type: Dict[str, dict]
            if default_client_settings and client_name in default_client_settings:
                client_with_defaults = default_client_settings[client_name]
            client_with_defaults.update(client_config)

            config = read_client_config(client_name, client_with_defaults)
            if config:
                parsed_configs.append(config)
        return parsed_configs
    else:
        raise ValueError("configs")
<commit_msg>Reuse self.document_highlight_scopes<commit_after>import sublime

PLUGIN_NAME = 'LSP'

try:
    from typing import List, Optional, Dict
    assert List and Optional and Dict
except ImportError:
    pass

def read_bool_setting(settings_obj: sublime.Settings, key: str, default: bool) -> bool:
    val = settings_obj.get(key)
    if isinstance(val, bool):
        return val
    else:
        return default

def read_dict_setting(settings_obj: sublime.Settings, key: str, default: dict) -> dict:
    val = settings_obj.get(key)
    if isinstance(val, dict):
        return val
    else:
        return default

def read_str_setting(settings_obj: sublime.Settings, key: str, default: str) -> str:
    val = settings_obj.get(key)
    if isinstance(val, str):
        return val
    else:
        return default

class Settings(object):

    def __init__(self):
        self.show_status_messages = True
        self.show_view_status = True
        self.auto_show_diagnostics_panel = True
        self.show_diagnostics_phantoms = False
        self.show_diagnostics_in_view_status = True
        self.only_show_lsp_completions = False
        self.diagnostics_highlight_style = "underline"
        self.document_highlight_style = "stippled"
        self.document_highlight_scopes = {
            "unknown": "text",
            "text": "text",
            "read": "markup.inserted",
            "write": "markup.changed"
        }
        self.diagnostics_gutter_marker = "dot"
        self.complete_all_chars = False
        self.completion_hint_type = "auto"
        self.resolve_completion_for_snippets = False
        self.log_debug = True
        self.log_server = True
        self.log_stderr = False
        self.log_payloads = False

    def update(self, settings_obj: sublime.Settings):
        self.show_status_messages = read_bool_setting(settings_obj, "show_status_messages", True)
        self.show_view_status = read_bool_setting(settings_obj, "show_view_status", True)
        self.auto_show_diagnostics_panel = read_bool_setting(settings_obj, "auto_show_diagnostics_panel", True)
        self.show_diagnostics_phantoms = read_bool_setting(settings_obj, "show_diagnostics_phantoms", False)
        self.show_diagnostics_in_view_status = read_bool_setting(settings_obj, "show_diagnostics_in_view_status", True)
        self.diagnostics_highlight_style = read_str_setting(settings_obj, "diagnostics_highlight_style", "underline")
        self.document_highlight_style = read_str_setting(settings_obj, "document_highlight_style", "stippled")
        self.document_highlight_scopes = read_dict_setting(settings_obj, "document_highlight_scopes",
                                                           self.document_highlight_scopes)
        self.diagnostics_gutter_marker = read_str_setting(settings_obj, "diagnostics_gutter_marker", "dot")
        self.only_show_lsp_completions = read_bool_setting(settings_obj, "only_show_lsp_completions", False)
        self.complete_all_chars = read_bool_setting(settings_obj, "complete_all_chars", True)
        self.completion_hint_type = read_str_setting(settings_obj, "completion_hint_type", "auto")
        self.resolve_completion_for_snippets = read_bool_setting(settings_obj, "resolve_completion_for_snippets", False)
        self.log_debug = read_bool_setting(settings_obj, "log_debug", False)
        self.log_server = read_bool_setting(settings_obj, "log_server", True)
        self.log_stderr = read_bool_setting(settings_obj, "log_stderr", False)
        self.log_payloads = read_bool_setting(settings_obj, "log_payloads", False)

class ClientConfigs(object):

    def __init__(self):
        self._default_settings = dict()  # type: Dict[str, dict]
        self._global_settings = dict()  # type: Dict[str, dict]
        self.defaults = []  # type: List[ClientConfig]
        self.all = []  # type: List[ClientConfig]

    def update(self, settings_obj: sublime.Settings):
        self._default_settings = read_dict_setting(settings_obj, "default_clients", {})
        self._global_settings = read_dict_setting(settings_obj, "clients", {})

        self.defaults = read_client_configs(self._default_settings)
        self.all = read_client_configs(self._global_settings, self._default_settings)

        client_enableds = list("=".join([config.name, str(config.enabled)]) for config in self.all)
        print(PLUGIN_NAME + ': global clients: ' + ", ".join(client_enableds))

    def _set_enabled(self, config_name: str, is_enabled: bool):
        if _settings_obj:
            client_settings = self._global_settings.setdefault(config_name, {})
            client_settings["enabled"] = is_enabled
            _settings_obj.set("clients", self._global_settings)
            sublime.save_settings("LSP.sublime-settings")

    def enable(self, config_name: str):
        self._set_enabled(config_name, True)

    def disable(self, config_name: str):
        self._set_enabled(config_name, False)

_settings_obj = None  # type: Optional[sublime.Settings]
settings = Settings()
client_configs = ClientConfigs()

def load_settings():
    global _settings_obj
    loaded_settings_obj = sublime.load_settings("LSP.sublime-settings")
    _settings_obj = loaded_settings_obj
    settings.update(loaded_settings_obj)
    client_configs.update(loaded_settings_obj)
    loaded_settings_obj.add_on_change("_on_new_settings", lambda: settings.update(loaded_settings_obj))
    loaded_settings_obj.add_on_change("_on_new_client_settings", lambda: client_configs.update(loaded_settings_obj))

def unload_settings():
    if _settings_obj:
        _settings_obj.clear_on_change("_on_new_settings")
        _settings_obj.clear_on_change("_on_new_client_settings")

class ClientConfig(object):
    def __init__(self, name, binary_args, scopes, syntaxes, languageId,
                 enabled=True, init_options=dict(), settings=dict(), env=dict()):
        self.name = name
        self.binary_args = binary_args
        self.scopes = scopes
        self.syntaxes = syntaxes
        self.languageId = languageId
        self.enabled = enabled
        self.init_options = init_options
        self.settings = settings
        self.env = env

def read_client_config(name, client_config):
    return ClientConfig(
        name,
        client_config.get("command", []),
        client_config.get("scopes", []),
        client_config.get("syntaxes", []),
        client_config.get("languageId", ""),
        client_config.get("enabled", True),
        client_config.get("initializationOptions", dict()),
        client_config.get("settings", dict()),
        client_config.get("env", dict())
    )

def read_client_configs(client_settings, default_client_settings=None) -> 'List[ClientConfig]':
    parsed_configs = []  # type: List[ClientConfig]
    if isinstance(client_settings, dict):
        for client_name, client_config in client_settings.items():

            # start with default settings for this client if available
            client_with_defaults = {}  # type: Dict[str, dict]
            if default_client_settings and client_name in default_client_settings:
                client_with_defaults = default_client_settings[client_name]
            client_with_defaults.update(client_config)

            config = read_client_config(client_name, client_with_defaults)
            if config:
                parsed_configs.append(config)
        return parsed_configs
    else:
        raise ValueError("configs")
<|endoftext|>