<commit_before>#!/usr/bin/env python
import subprocess
import logging
import sys
import os
from os.path import join

MAIN_DIR = os.path.expanduser('~/.dakis')
JOBS_DIR = join(MAIN_DIR, 'jobs')

def prepare_environment():
    '''Prepares ~/.dakis directory and installs Python packages'''
    subprocess.Popen('mkdir -p %s && mv %s %s' % (JOBS_DIR, sys.argv[0], MAIN_DIR), shell=True)
    cmds = [
        'cd %s' % MAIN_DIR,
        'wget https://bootstrap.pypa.io/get-pip.py',
        'python get-pip.py --user',
        'rm get-pip.py',
        'pip install --user requests argparse || ~/.local/bin/pip install --user requests argparse',
    ]
    subprocess.Popen(' && '.join(cmds), shell=True)
    p1 = subprocess.Popen(' && '.join(cmds), shell=True)
    p1.communicate()  # TODO: replace with p1.wait()
    print("\n == Successfully prepared Dakis environment == \n")

def run_next_task(exp_id, executable, use_job=False):
    url = 'http://dakis.gimbutas.lt/api/exp/%d/next-task/' % exp_id
    resp = requests.get(url)
    task = resp.json()
    logging.info('Getting next task: exp_id=%d, status_code=%d, resp=%s' % (exp_id, resp.status_code, resp.json()))
    if resp.status_code == 200 and resp.json():
        cmd = [
            '%s' % executable,       # Should pass path as it will be called  .strip('./')
            '--gkls_cls=%s' % task['func_cls'],
            '--gkls_fid=%s' % task['func_id'],
            '--task_id=%s' % task['id'],
            '--callback=%s' % join(MAIN_DIR, 'worker.py'),
        ]
        try:
            if use_job:
                job_filename = join(JOBS_DIR, 'e%dc%df%d.sh' % (exp_id, task['func_cls'], task['func_id']))
                job_file = open(job_filename, 'w')
                job_file.write('#!/bin/bash\n#$ -j y\n#$ -S /bin/bash\n#$ -cwd\nmpirun -np 1 ')  # Header
                job_file.write(' '.join(cmd) + '\n')
                job_file.close()
                logging.info('Created job file: %s' % job_filename)
                add_to_queue_cmd = 'qsub -pe orte 1 -o {0}.o -e {0}.e {0}'.format(job_filename)
                # Specify stdout and stderr paths for qsub and remove them after execution
                print('Calling command:', add_to_queue_cmd)
                p1 = subprocess.Popen(add_to_queue_cmd, shell=True, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE)  # , close_fds=True)
                print('Add to queue command response:', p1.communicate())  # Error: cannot submit qsub job using ssh
                logging.info('Called command: %s' % add_to_queue_cmd)
            else:
                logging.info('Calling: cmd=%s' % ' '.join(cmd))
                subprocess.Popen(' '.join(cmd), shell=True, stdin=None, stdout=None, stderr=None)
        except Exception as e:
            logging.error('Got error while calling:  %s' % str(e))

def send_task_results(args):
    url = 'http://dakis.gimbutas.lt/api/tasks/%d/' % args.task_id
    data = {
        'calls': args.calls,
        'duration': args.duration,
        'subregions': args.subregions,
        'status': args.status,
    }
    logging.info('Sending: url=%s, data=%s' % (url, data))
    resp = requests.put(url, data)
    logging.info('Response from sending: status_code=%s, data=%s' % (resp.status_code, resp.json()))
    return resp.json()

def prepare_executable(args):
    '''Clone and compile experiments code. Uses REST API if not all data provided in parameters.'''
    exp_id = args.exp_id
    if args.repository and args.branch and args.executable:
        repository = args.repository
        branch = args.branch
        exe_file = args.executable
    else:
        url = 'http://dakis.gimbutas.lt/api/experiments/%d/' % exp_id
        resp = requests.get(url)
        exp = resp.json()
        repository = exp['repository']
        branch = exp['branch']
        exe_file = exp['executable']

    exe_dir = join(MAIN_DIR, 'exp_%d' % exp_id)
    executable = join(exe_dir, exe_file)

    if not os.path.exists(executable):  # Note: code version will be cached, new commits won't be pulled
        if not os.path.exists(exe_dir):
            cmd = 'git clone {0} {1} && cd {1} && git fetch origin {2} && git checkout {2} && git pull -r && make'.format(repository, exe_dir, branch)
            proc = subprocess.Popen(cmd, shell=True)
            proc.communicate()
        else:
            proc = subprocess.Popen('cd %s && git pull -r && make' % exe_dir, shell=True)
            proc.communicate()
    return executable

def request_to_run_next_task(exp_id):
    url = 'http://dakis.gimbutas.lt/api/exp/%d/run/' % exp_id
    resp = requests.get(url)
    return

def get_argparser():
    import argparse
    parser = argparse.ArgumentParser(description='Schedules tasks')
    parser.add_argument('-exp', '--exp_id', type=int, help='Experiment ID', nargs='?', default=None)
    parser.add_argument('-exe', '--executable', type=str, help='Executable file', nargs='?', default=None)
    parser.add_argument('-rep', '--repository', type=str, help='Source code repository', nargs='?', default=None)
    parser.add_argument('-br', '--branch', type=str, help='Repository branch', nargs='?', default=None)

    # Worker should find the job by itself.
    parser.add_argument('-j', '--job', help='Create job and execute task through supercomputer task queue', nargs='?', const=True)

    parser.add_argument('-task', '--task_id', type=int, help='Task ID', nargs=None, default=None)
    parser.add_argument('-calls', '--calls', type=int, help='Number of calls', nargs=None, default=None)
    parser.add_argument('-duration', '--duration', type=float, help='Execution duration', nargs=None, default=None)
    parser.add_argument('-subs', '--subregions', type=int, help='Number of subregions', nargs=None, default=None)
    parser.add_argument('-st', '--status', type=str, help='Status of the task. D - done, S - suspended.', nargs=None, default=None)

    # Prepare environment arguments
    parser.add_argument('-env', '--prepare_environment', help='Create ~/.dakis dir and prepare environment', nargs='?', const=True)
    return parser

def main():
    global requests
    import requests
    args = get_argparser().parse_args()

    logging.basicConfig(
        level=logging.INFO,
        filename=os.path.expanduser(join(MAIN_DIR, 'worker.log')),
        format='%(asctime)s - %(levelname)s - %(message)s',
    )
    logging.info('Invoked with argv: %s' % sys.argv)
    logging.info('Invoked with: exp_id=%s executable=%s task_id=%s calls=%s -duration=%s --subregions=%s --status=%s --job=%s' % (
                 args.exp_id, args.executable, args.task_id, args.calls, args.duration, args.subregions, args.status, args.job))

    if args.exp_id:
        exe = prepare_executable(args)
        run_next_task(args.exp_id, exe, args.job)
    elif args.task_id:
        # Send results
        resp_json = send_task_results(args)
        # Remove job file if it exists
        exp_id = int(resp_json['experiment'].split('experiments')[-1].strip('/'))
        job_filename = join(JOBS_DIR, 'e%dc%df%d.sh' % (exp_id, resp_json['func_cls'], resp_json['func_id']))
        if os.path.isfile(job_filename):
            os.remove(job_filename)
        for file in os.listdir(JOBS_DIR):
            if file.endswith('.o') or file.endswith('.e'):
                os.remove(join(JOBS_DIR, file))
        request_to_run_next_task(exp_id)

if __name__ == '__main__':
    if (not os.path.exists(MAIN_DIR) or not os.path.exists(JOBS_DIR)
        or 'env' in sys.argv or '-env' in sys.argv or '--prepare_environment' in sys.argv):
        prepare_environment()
    else:
        main()
<commit_msg>Worker require at least 12 hours for job execution<commit_after>#!/usr/bin/env python
import subprocess
import logging
import sys
import os
from os.path import join

MAIN_DIR = os.path.expanduser('~/.dakis')
JOBS_DIR = join(MAIN_DIR, 'jobs')

def prepare_environment():
    '''Prepares ~/.dakis directory and installs Python packages'''
    subprocess.Popen('mkdir -p %s && mv %s %s' % (JOBS_DIR, sys.argv[0], MAIN_DIR), shell=True)
    cmds = [
        'cd %s' % MAIN_DIR,
        'wget https://bootstrap.pypa.io/get-pip.py',
        'python get-pip.py --user',
        'rm get-pip.py',
        'pip install --user requests argparse || ~/.local/bin/pip install --user requests argparse',
    ]
    subprocess.Popen(' && '.join(cmds), shell=True)
    p1 = subprocess.Popen(' && '.join(cmds), shell=True)
    p1.communicate()  # TODO: replace with p1.wait()
    print("\n == Successfully prepared Dakis environment == \n")

def run_next_task(exp_id, executable, use_job=False):
    url = 'http://dakis.gimbutas.lt/api/exp/%d/next-task/' % exp_id
    resp = requests.get(url)
    task = resp.json()
    logging.info('Getting next task: exp_id=%d, status_code=%d, resp=%s' % (exp_id, resp.status_code, resp.json()))
    if resp.status_code == 200 and resp.json():
        cmd = [
            '%s' % executable,       # Should pass path as it will be called  .strip('./')
            '--gkls_cls=%s' % task['func_cls'],
            '--gkls_fid=%s' % task['func_id'],
            '--task_id=%s' % task['id'],
            '--callback=%s' % join(MAIN_DIR, 'worker.py'),
        ]
        try:
            if use_job:
                job_filename = join(JOBS_DIR, 'e%dc%df%d.sh' % (exp_id, task['func_cls'], task['func_id']))
                job_file = open(job_filename, 'w')
                job_file.write('#!/bin/bash\n#$ -j y\n#$ -l h_rt=12:00:00\n#$ -S /bin/bash\n#$ -cwd\nmpirun -np 1 ')  # Header
                job_file.write(' '.join(cmd) + '\n')
                job_file.close()
                logging.info('Created job file: %s' % job_filename)
                add_to_queue_cmd = 'qsub -pe orte 1 -o {0}.o -e {0}.e {0}'.format(job_filename)
                # Specify stdout and stderr paths for qsub and remove them after execution
                print('Calling command:', add_to_queue_cmd)
                p1 = subprocess.Popen(add_to_queue_cmd, shell=True, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE)  # , close_fds=True)
                print('Add to queue command response:', p1.communicate())  # Error: cannot submit qsub job using ssh
                logging.info('Called command: %s' % add_to_queue_cmd)
            else:
                logging.info('Calling: cmd=%s' % ' '.join(cmd))
                subprocess.Popen(' '.join(cmd), shell=True, stdin=None, stdout=None, stderr=None)
        except Exception as e:
            logging.error('Got error while calling:  %s' % str(e))

def send_task_results(args):
    url = 'http://dakis.gimbutas.lt/api/tasks/%d/' % args.task_id
    data = {
        'calls': args.calls,
        'duration': args.duration,
        'subregions': args.subregions,
        'status': args.status,
    }
    logging.info('Sending: url=%s, data=%s' % (url, data))
    resp = requests.put(url, data)
    logging.info('Response from sending: status_code=%s, data=%s' % (resp.status_code, resp.json()))
    return resp.json()

def prepare_executable(args):
    '''Clone and compile experiments code. Uses REST API if not all data provided in parameters.'''
    exp_id = args.exp_id
    if args.repository and args.branch and args.executable:
        repository = args.repository
        branch = args.branch
        exe_file = args.executable
    else:
        url = 'http://dakis.gimbutas.lt/api/experiments/%d/' % exp_id
        resp = requests.get(url)
        exp = resp.json()
        repository = exp['repository']
        branch = exp['branch']
        exe_file = exp['executable']

    exe_dir = join(MAIN_DIR, 'exp_%d' % exp_id)
    executable = join(exe_dir, exe_file)

    if not os.path.exists(executable):  # Note: code version will be cached, new commits won't be pulled
        if not os.path.exists(exe_dir):
            cmd = 'git clone {0} {1} && cd {1} && git fetch origin {2} && git checkout {2} && git pull -r && make'.format(repository, exe_dir, branch)
            proc = subprocess.Popen(cmd, shell=True)
            proc.communicate()
        else:
            proc = subprocess.Popen('cd %s && git pull -r && make' % exe_dir, shell=True)
            proc.communicate()
    return executable

def request_to_run_next_task(exp_id):
    url = 'http://dakis.gimbutas.lt/api/exp/%d/run/' % exp_id
    resp = requests.get(url)
    return

def get_argparser():
    import argparse
    parser = argparse.ArgumentParser(description='Schedules tasks')
    parser.add_argument('-exp', '--exp_id', type=int, help='Experiment ID', nargs='?', default=None)
    parser.add_argument('-exe', '--executable', type=str, help='Executable file', nargs='?', default=None)
    parser.add_argument('-rep', '--repository', type=str, help='Source code repository', nargs='?', default=None)
    parser.add_argument('-br', '--branch', type=str, help='Repository branch', nargs='?', default=None)

    # Worker should find the job by itself.
    parser.add_argument('-j', '--job', help='Create job and execute task through supercomputer task queue', nargs='?', const=True)

    parser.add_argument('-task', '--task_id', type=int, help='Task ID', nargs=None, default=None)
    parser.add_argument('-calls', '--calls', type=int, help='Number of calls', nargs=None, default=None)
    parser.add_argument('-duration', '--duration', type=float, help='Execution duration', nargs=None, default=None)
    parser.add_argument('-subs', '--subregions', type=int, help='Number of subregions', nargs=None, default=None)
    parser.add_argument('-st', '--status', type=str, help='Status of the task. D - done, S - suspended.', nargs=None, default=None)

    # Prepare environment arguments
    parser.add_argument('-env', '--prepare_environment', help='Create ~/.dakis dir and prepare environment', nargs='?', const=True)
    return parser

def main():
    global requests
    import requests
    args = get_argparser().parse_args()

    logging.basicConfig(
        level=logging.INFO,
        filename=os.path.expanduser(join(MAIN_DIR, 'worker.log')),
        format='%(asctime)s - %(levelname)s - %(message)s',
    )
    logging.info('Invoked with argv: %s' % sys.argv)
    logging.info('Invoked with: exp_id=%s executable=%s task_id=%s calls=%s -duration=%s --subregions=%s --status=%s --job=%s' % (
                 args.exp_id, args.executable, args.task_id, args.calls, args.duration, args.subregions, args.status, args.job))

    if args.exp_id:
        exe = prepare_executable(args)
        run_next_task(args.exp_id, exe, args.job)
    elif args.task_id:
        # Send results
        resp_json = send_task_results(args)
        # Remove job file if it exists
        exp_id = int(resp_json['experiment'].split('experiments')[-1].strip('/'))
        job_filename = join(JOBS_DIR, 'e%dc%df%d.sh' % (exp_id, resp_json['func_cls'], resp_json['func_id']))
        if os.path.isfile(job_filename):
            os.remove(job_filename)
        for file in os.listdir(JOBS_DIR):
            if file.endswith('.o') or file.endswith('.e'):
                os.remove(join(JOBS_DIR, file))
        request_to_run_next_task(exp_id)

if __name__ == '__main__':
    if (not os.path.exists(MAIN_DIR) or not os.path.exists(JOBS_DIR)
        or 'env' in sys.argv or '-env' in sys.argv or '--prepare_environment' in sys.argv):
        prepare_environment()
    else:
        main()
<|endoftext|>


<commit_before>import argparse
import os
import sys
from scm.plams import KFReader, Settings, ADFJob, init, finish

def main():
    """
        Main interface that reads the file and retrieve all useful geoms etc.
        Then rewrite geometries in a usable formatter_class
        Prep all NBO computations
        Run ADF on each computation
    """
    # Retrieve command line values
    args = get_input_arguments()
    input_file = args['input_file']

    # Init PLAMS, Setting up a plams.$PID directory in the working dir
    init()

    # Retrieve settings from input file
    settings = get_settings(args)

    geometries = IRC_coordinates_from_t21(input_file)
    natoms = number_of_atoms(input_file)

    # Prep a bunch of NBO computations
    NBO_jobs = [prepare_NBO_computation(geom, settings) for geom in geometries]
    # Run each job in parallel as managed by plams
    NBO_results = [job.run() for job in NBO_jobs]
    # NBO_values is a list of list of charges
    NBO_values = [extract_NBO_charges(output, natoms) for output in NBO_results]
    # Write NBO data
    with open('NBOCharges.data', mode='w+') as output_file:
        for i in range(0, len(NBO_values)):
            output_file.write('     '.join(NBO_values[i])) + '\n'

    # Close plams session
    finish()

def IRC_coordinates_to_xyz_file(filename, geometries):
    """
        Export coordinates in geometries table to a file 'filename'
        straight in the working directory
    """
    with open(filename, mode='w+') as XYZ:
        print(XYZ.name)
        for i in range(0, len(geometries)):
            XYZ.write('New molecule\n')
            for j in range(0, len(geometries[i])):
                for k in range(0, len(geometries[i][j])):
                    XYZ.write(str(geometries[i][j][k]) + '       ')
                XYZ.write('\n')
            XYZ.write('\n\n')
    return

def IRC_coordinates_from_t21(input_file):
    """
        Extract all data from a TAPE21 file.
    """
    # Read TAPE21 and get all useful data
    t21 = KFReader(input_file)

    # Number of atoms: 7
    natoms = t21.read('Geometry', 'nr of atoms')
    # atom types as indexes: [1, 2, 2, 3, 3, 4, 3]
    aatoms = t21.read('Geometry', 'fragment and atomtype index')[natoms:]
    # Atom symbols as list: ['C', 'O', 'H', 'B']
    xatoms = str(t21.read('Geometry', 'atomtype')).split()
    # Actual list of atoms as used in geometry: ['C', 'O', 'O', 'H', 'H', 'B', 'H']
    satoms = [xatoms[aatoms[order - 1] - 1] for order in t21.read('Geometry', 'atom order index')[:natoms]]

    nstep_fw = t21.read('IRC_Forward', 'CurrentPoint')
    nstep_bw = t21.read('IRC_Backward', 'CurrentPoint')
    geometries_fw = t21.read('IRC_Forward', 'xyz')[0:nstep_fw * natoms * 3]
    geometries_bw = t21.read('IRC_Backward', 'xyz')[0:nstep_bw * natoms * 3]
    geometries_init = t21.read('IRC', 'xyz')

    # Reformat geometries into a long list of geometries for each step
    geometries_bw = coordinates_from_list(geometries_bw, natoms)
    geometries_fw = coordinates_from_list(geometries_fw, natoms)
    geometries_init = coordinates_from_list(geometries_init, natoms)
    geometries_fw.reverse()

    geometries = geometries_fw + geometries_init + geometries_bw

    return [[[s, mol[0], mol[1], mol[2]] for i, (s, mol) in enumerate(zip(satoms, molecule))] for molecule in geometries]

def coordinates_from_list(coordinates_list, natoms):
    """
        Rewrites a list into a list of lists of lists:
        [x1,y1,z1,x2,y2,z2,x'1,y'1,z'1,x'2,y'2,z'2] becomes:
        [ [ [x1,y1,z1], [x2,y2,z2]], [ [x'1,y'1,z'1], [x'2,y'2,z'2] ] ]
        which can be acessed as:
        list[moleculeNumber][atomNumber][coordinateNumber]
    """
    # list = [x1,y1,z1,x2,y2,z2,x'1,y'1,z'1,x'2,y'2,z'2]
    coordinates_list = list(list_chunks(coordinates_list, 3))
    # [[x1,y1,z1],[x2,y2,z2],[x'1,y'1,z'1],[x'2,y'2,z'2]]
    coordinates_list = list(list_chunks(coordinates_list, natoms))
    # [ [[x1,y1,z1],[x2,y2,z2]], [[x'1,y'1,z'1],[x'2,y'2,z'2]]]
    return coordinates_list

def list_chunks(list, n):
    """
        Generator that splits a list in chunks of size n
        /!\ Does not care about the end of the list if len(list) is not a multiple of n
    """
    for i in range(0, len(list), n):
        yield list[i:i + n]

def prepare_NBO_computation(geometry, runparameters):
    """
        Taking a geometry and a set of ADF parameters (Basis set, Functional, ZORA, etc)
        Add the NBO Necessary keywords and returns the plams job
    """
    nbo_script = '\n'.join(['"$ADFBIN/adfnbo" << eor',
                            'write',
                            'spherical',
                            'nbo-analysis',
                            'eor',
                            '',
                            '"$ADFBIN/gennbo6" FILE47',
                            '',
                            '"$ADFBIN/adfnbo" <<eor',
                            'spherical',
                            'fock',
                            'read',
                            'end input',
                            'eor'])
    runparameters.runscript.post = nbo_script
    job = ADFJob(name='NBO Computation', settings=runparameters)
    print(job.get_runscript())
    return job

def extract_NBO_charges(output, natoms):
    """
        extract NBO Charges parsing the outFile
    """
    # Initialize charges list
    charges = []

    with open(output, mode='r') as outFile:
        line = 'Foobar line'
        while line:
            line = outFile.readline()
            if line.contains('Summary of Natural Population Analysis:'):
                # We have the table we want for the charges
                # Read five lines to remove the header:
                # Summary of Natural Population Analysis:
                #
                # Natural Population
                # Natural    ---------------------------------------------
                # Atom No    Charge        Core      Valence    Rydberg      Total
                # ----------------------------------------------------------------
                for i in range(0, 5):
                    outFile.readline()
                # Then we read the actual table:
                for i in range(0, natoms):
                    # Each line follow the header with the form:
                    # C  1    0.92349      1.99948     3.03282    0.04422     5.07651
                    line = outFile.readline()
                    line = line.split(' ')
                    charges.append(line[2])
                # We have reached the end of the table, we can break the while loop
                break
        return charges

def number_of_atoms(input_file):
    """
        Extracts natoms from TAPE21
    """
    t21 = KFReader(input_file)
    return t21.read('Geometry', 'nr of atoms')

def get_input_arguments():
    """
        Check command line options and accordingly set computation parameters
    """
    parser = argparse.ArgumentParser(description=help_description(),
                                     epilog=help_epilog())
    parser.formatter_class = argparse.RawDescriptionHelpFormatter
    parser.add_argument('input_file', type=str, nargs=1,
                        help='TAPE21 Containg IRC path')
    parser.add_argument('-f', '--functional', type=str, nargs='?', default='BLYP-D3',
                        help='Functional used for the computation, as BLYP-D3 or BP86\n'
                             'Hyphen will split into functional/dispersion parts when applicable')
    parser.add_argument('-r', '--relativistic', type=str, nargs='?', default='None',
                        help='Relativistic effects: Scalar, Spin-Orbit or None (default)')
    parser.add_argument('-b', '--basisset', type=str, nargs='?', default='DZP',
                        help='The basis set to use for all atoms')
    parser.add_argument('-c', '--frozencore', type=str, nargs='?', default='None',
                        help='Frozen core to use: None (Default), Small, Large')
    parser.add_argument('-i', '--integrationquality', type=str, nargs='?', default='Good',
                        help='Numerical Integration Quality. Default: Good')
    try:
        args = parser.parse_args()
    except argparse.ArgumentError as error:
        print(str(error))  # Print something like "option -a not recognized"
        sys.exit(2)

    # Get values from parser
    values = dict.fromkeys(['input_file', 'functional', 'dispersion', 'relativistic',
                            'basisset', 'frozencore', 'integrationquality'])
    values['input_file'] = os.path.basename(args.input_file[0])
    print(args)
    functional = args.functional.split('-')
    print(functional)
    values['functional'] = functional[0]
    if len(functional) > 1:
        if functional[1] == 'GD3' or functional[1] == 'D3':
            values['dispersion'] = 'Grimme3'
        elif functional[1] == 'GD3BJ':
            values['dispersion'] = 'Grimme3 BJDAMP'
    else:
        values['dispersion'] = None
    values['relativistic'] = args.relativistic
    values['basisset'] = args.basisset
    values['frozencore'] = args.frozencore
    values['integrationquality'] = args.integrationquality
    print(values)
    return values

def get_settings(values):
    """
        Retrieve settings from command line, and set them up
    """
    settings = Settings()
    settings.input.XC.GGA = values['functional']
    if values['dispersion'] is not None:
        settings.input.XC.DISPERSION = values['dispersion']
    settings.input.BASIS.type = values['basisset']
    settings.input.BASIS.core = values['frozencore']
    settings.input.BASIS.createoutput = 'None'
    settings.input.NumericalQuality = values['integrationquality']
    settings.input.RELATIVISTIC = values['relativistic'] + " ZORA"
    settings.input.AOMAT2FILE = ''
    settings.input.SAVE = 'TAPE15'
    settings.input.FULLFOCK = ''
    settings.input.NOPRINT = "LOGFILE"

    print(settings)
    return settings

def help_description():
    """
        Returns description of program for help message
    """
    return 'Help Description // To fill'

def help_epilog():
    """
        Returns additionnal help message
    """
    return 'Help epilog // To Fill'

if __name__ == '__main__':
    main()
<commit_msg>Manage NBO charges printing to file Not extremely clean but will work<commit_after>import argparse
import os
import sys
from scm.plams import KFReader, Settings, ADFJob, init, finish

def main():
    """
        Main interface that reads the file and retrieve all useful geoms etc.
        Then rewrite geometries in a usable formatter_class
        Prep all NBO computations
        Run ADF on each computation
    """
    # Retrieve command line values
    args = get_input_arguments()
    input_file = args['input_file']
    output_file = args['output_file']

    # Init PLAMS, Setting up a plams.$PID directory in the working dir
    init()

    # Retrieve settings from input file
    settings = get_settings(args)

    geometries = IRC_coordinates_from_t21(input_file)
    natoms = number_of_atoms(input_file)

    # Prep a bunch of NBO computations
    NBO_jobs = [prepare_NBO_computation(geom, settings) for geom in geometries]
    # Run each job in parallel as managed by plams
    NBO_results = [job.run() for job in NBO_jobs]
    # NBO_values is a list of list of charges
    NBO_values = [extract_NBO_charges(output, natoms) for output in NBO_results]
    # Write NBO data
    print_NBO_charges_to_file(NBO_values, output_file)

    # Close plams session
    finish()

def IRC_coordinates_to_xyz_file(filename, geometries):
    """
        Export coordinates in geometries table to a file 'filename'
        straight in the working directory
    """
    with open(filename, mode='w+') as XYZ:
        print(XYZ.name)
        for i in range(0, len(geometries)):
            XYZ.write('New molecule\n')
            for j in range(0, len(geometries[i])):
                for k in range(0, len(geometries[i][j])):
                    XYZ.write(str(geometries[i][j][k]) + '       ')
                XYZ.write('\n')
            XYZ.write('\n\n')
    return

def IRC_coordinates_from_t21(input_file):
    """
        Extract all data from a TAPE21 file.
    """
    # Read TAPE21 and get all useful data
    t21 = KFReader(input_file)

    # Number of atoms: 7
    natoms = t21.read('Geometry', 'nr of atoms')
    # atom types as indexes: [1, 2, 2, 3, 3, 4, 3]
    aatoms = t21.read('Geometry', 'fragment and atomtype index')[natoms:]
    # Atom symbols as list: ['C', 'O', 'H', 'B']
    xatoms = str(t21.read('Geometry', 'atomtype')).split()
    # Actual list of atoms as used in geometry: ['C', 'O', 'O', 'H', 'H', 'B', 'H']
    satoms = [xatoms[aatoms[order - 1] - 1] for order in t21.read('Geometry', 'atom order index')[:natoms]]

    nstep_fw = t21.read('IRC_Forward', 'CurrentPoint')
    nstep_bw = t21.read('IRC_Backward', 'CurrentPoint')
    geometries_fw = t21.read('IRC_Forward', 'xyz')[0:nstep_fw * natoms * 3]
    geometries_bw = t21.read('IRC_Backward', 'xyz')[0:nstep_bw * natoms * 3]
    geometries_init = t21.read('IRC', 'xyz')

    # Reformat geometries into a long list of geometries for each step
    geometries_bw = coordinates_from_list(geometries_bw, natoms)
    geometries_fw = coordinates_from_list(geometries_fw, natoms)
    geometries_init = coordinates_from_list(geometries_init, natoms)
    geometries_fw.reverse()

    geometries = geometries_fw + geometries_init + geometries_bw

    return [[[s, mol[0], mol[1], mol[2]] for i, (s, mol) in enumerate(zip(satoms, molecule))] for molecule in geometries]

def coordinates_from_list(coordinates_list, natoms):
    """
        Rewrites a list into a list of lists of lists:
        [x1,y1,z1,x2,y2,z2,x'1,y'1,z'1,x'2,y'2,z'2] becomes:
        [ [ [x1,y1,z1], [x2,y2,z2]], [ [x'1,y'1,z'1], [x'2,y'2,z'2] ] ]
        which can be acessed as:
        list[moleculeNumber][atomNumber][coordinateNumber]
    """
    # list = [x1,y1,z1,x2,y2,z2,x'1,y'1,z'1,x'2,y'2,z'2]
    coordinates_list = list(list_chunks(coordinates_list, 3))
    # [[x1,y1,z1],[x2,y2,z2],[x'1,y'1,z'1],[x'2,y'2,z'2]]
    coordinates_list = list(list_chunks(coordinates_list, natoms))
    # [ [[x1,y1,z1],[x2,y2,z2]], [[x'1,y'1,z'1],[x'2,y'2,z'2]]]
    return coordinates_list

def list_chunks(list, n):
    """
        Generator that splits a list in chunks of size n
        /!\ Does not care about the end of the list if len(list) is not a multiple of n
    """
    for i in range(0, len(list), n):
        yield list[i:i + n]

def prepare_NBO_computation(geometry, runparameters):
    """
        Taking a geometry and a set of ADF parameters (Basis set, Functional, ZORA, etc)
        Add the NBO Necessary keywords and returns the plams job
    """
    nbo_script = '\n'.join(['"$ADFBIN/adfnbo" << eor',
                            'write',
                            'spherical',
                            'nbo-analysis',
                            'eor',
                            '',
                            '"$ADFBIN/gennbo6" FILE47',
                            '',
                            '"$ADFBIN/adfnbo" <<eor',
                            'spherical',
                            'fock',
                            'read',
                            'end input',
                            'eor'])
    runparameters.runscript.post = nbo_script
    job = ADFJob(name='NBO Computation', settings=runparameters)
    print(job.get_runscript())
    return job

def extract_NBO_charges(output, natoms):
    """
        extract NBO Charges parsing the outFile
    """
    # Initialize charges list
    charges = []

    with open(output, mode='r') as outFile:
        line = 'Foobar line'
        while line:
            line = outFile.readline()
            if 'Summary of Natural Population Analysis:' in line:
                # We have the table we want for the charges
                # Read five lines to remove the header:
                # Summary of Natural Population Analysis:
                #
                # Natural Population
                # Natural    ---------------------------------------------
                # Atom No    Charge        Core      Valence    Rydberg      Total
                # ----------------------------------------------------------------
                for i in range(0, 5):
                    outFile.readline()
                # Then we read the actual table:
                for i in range(0, natoms):
                    # Each line follow the header with the form:
                    # C  1    0.92349      1.99948     3.03282    0.04422     5.07651
                    line = outFile.readline()
                    line = line.split()
                    charges.append(line[2])
                # We have reached the end of the table, we can break the while loop
                break
        return charges

def print_NBO_charges_to_file(charges_list, file):
    """
        Export NBO charges to a file that one can import in a spreadsheet or gnuplot
    """
    with open(file, mode='w+') as output_file:
        for i in range(0, len(charges_list)):
            output_file.write('     '.join(charges_list[i]) + '\n')

def number_of_atoms(input_file):
    """
        Extracts natoms from TAPE21
    """
    t21 = KFReader(input_file)
    return t21.read('Geometry', 'nr of atoms')

def get_input_arguments():
    """
        Check command line options and accordingly set computation parameters
    """
    parser = argparse.ArgumentParser(description=help_description(),
                                     epilog=help_epilog())
    parser.formatter_class = argparse.RawDescriptionHelpFormatter
    parser.add_argument('input_file', type=str, nargs=1,
                        help='TAPE21 Containg IRC path')
    parser.add_argument('output_file', type=str, nargs=1,
                        help='Output file in which to print the NBO charges')
    parser.add_argument('-f', '--functional', type=str, nargs='?', default='BLYP-D3',
                        help='Functional used for the computation, as BLYP-D3 or BP86\n'
                             'Hyphen will split into functional/dispersion parts when applicable')
    parser.add_argument('-r', '--relativistic', type=str, nargs='?', default='None',
                        help='Relativistic effects: Scalar, Spin-Orbit or None (default)')
    parser.add_argument('-b', '--basisset', type=str, nargs='?', default='DZP',
                        help='The basis set to use for all atoms')
    parser.add_argument('-c', '--frozencore', type=str, nargs='?', default='None',
                        help='Frozen core to use: None (Default), Small, Large')
    parser.add_argument('-i', '--integrationquality', type=str, nargs='?', default='Good',
                        help='Numerical Integration Quality. Default: Good')
    try:
        args = parser.parse_args()
    except argparse.ArgumentError as error:
        print(str(error))  # Print something like "option -a not recognized"
        sys.exit(2)

    # Get values from parser
    values = dict.fromkeys(['input_file', 'functional', 'dispersion', 'relativistic',
                            'basisset', 'frozencore', 'integrationquality'])
    values['input_file'] = os.path.basename(args.input_file[0])
    print(args)
    functional = args.functional.split('-')
    print(functional)
    values['functional'] = functional[0]
    if len(functional) > 1:
        if functional[1] == 'GD3' or functional[1] == 'D3':
            values['dispersion'] = 'Grimme3'
        elif functional[1] == 'GD3BJ':
            values['dispersion'] = 'Grimme3 BJDAMP'
    else:
        values['dispersion'] = None
    values['relativistic'] = args.relativistic
    values['basisset'] = args.basisset
    values['frozencore'] = args.frozencore
    values['integrationquality'] = args.integrationquality
    print(values)
    return values

def get_settings(values):
    """
        Retrieve settings from command line, and set them up
    """
    settings = Settings()
    settings.input.XC.GGA = values['functional']
    if values['dispersion'] is not None:
        settings.input.XC.DISPERSION = values['dispersion']
    settings.input.BASIS.type = values['basisset']
    settings.input.BASIS.core = values['frozencore']
    settings.input.BASIS.createoutput = 'None'
    settings.input.NumericalQuality = values['integrationquality']
    settings.input.RELATIVISTIC = values['relativistic'] + " ZORA"
    settings.input.AOMAT2FILE = ''
    settings.input.SAVE = 'TAPE15'
    settings.input.FULLFOCK = ''
    settings.input.NOPRINT = "LOGFILE"

    print(settings)
    return settings

def help_description():
    """
        Returns description of program for help message
    """
    return 'Help Description // To fill'

def help_epilog():
    """
        Returns additionnal help message
    """
    return 'Help epilog // To Fill'

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import os
import settings
from django.core.management.base import BaseCommand
from xbrowse_server.base.models import Project, Individual
from django.core.exceptions import ObjectDoesNotExist

class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument('args', nargs='*')

    def handle(self, *args, **options):
        """Command line args:
            arg0 = project id
            arg1 = file path - where file is a tsv table with 2 columns:
                    column0: the individual ID
                    column1: bam file path relative to the base settings.READ_VIZ_BAM_PATH url or directory - so that
                        the absolute bam path can be derived by doing os.path.join(settings.READ_VIZ_BAM_PATH, column1)
        """
        project_id = args[0]
        project = Project.objects.get(project_id=project_id)

        for line in open(args[1]).readlines():
            try:
                indiv_id, bam_path = line.strip('\n').split('\t')
            except Exception as e:
                raise ValueError("Couldn't parse line: %s" % line, e)

            try:
                indiv = Individual.objects.get(project=project, indiv_id=indiv_id)
            except ObjectDoesNotExist as e:
                print("ERROR: Individual not found in xBrowse: '%s'. Skipping.." % indiv_id)
                continue

            absolute_path = os.path.join(settings.READ_VIZ_BAM_PATH, bam_path)
            if absolute_path.startswith('http'):
                pass
            elif not os.path.isfile(absolute_path):
                print("ERROR: " + absolute_path + " not found. Skipping..")
                continue
            indiv.bam_file_path = bam_path
            indiv.save()

<commit_msg>Added check to make sure bam paths are valid<commit_after>import os
import settings
from django.core.management.base import BaseCommand
from xbrowse_server.base.models import Project, Individual
from django.core.exceptions import ObjectDoesNotExist

class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument('args', nargs='*')

    def handle(self, *args, **options):
        """Command line args:
            arg0 = project id
            arg1 = file path - where file is a tsv table with 2 columns:
                    column0: the individual ID
                    column1: bam file path relative to the base settings.READ_VIZ_BAM_PATH url or directory - so that
                        the absolute bam path can be derived by doing os.path.join(settings.READ_VIZ_BAM_PATH, column1)
        """
        project_id = args[0]
        project = Project.objects.get(project_id=project_id)

        for line in open(args[1]).readlines():
            try:
                indiv_id, bam_path = line.strip('\n').split('\t')
            except Exception as e:
                raise ValueError("Couldn't parse line: %s" % line, e)

            try:
                indiv = Individual.objects.get(project=project, indiv_id=indiv_id)
            except ObjectDoesNotExist as e:
                print("ERROR: Individual not found in xBrowse: '%s'. Skipping.." % indiv_id)
                continue

            absolute_path = os.path.join(settings.READ_VIZ_BAM_PATH, bam_path)
            if absolute_path.startswith('http'):
                response = requests.request("GET", absolute_path, auth=(settings.READ_VIZ_USERNAME, settings.READ_VIZ_PASSWD), verify=False)
                if response.status_code != 200:
                    print("ERROR: " + absolute_path + " reponse code == " + str(response.status_code) + ". Skipping..")
                    continue
            elif not os.path.isfile(absolute_path):
                print("ERROR: " + absolute_path + " not found. Skipping..")
                continue
            indiv.bam_file_path = bam_path
            indiv.save()

<|endoftext|>


<commit_before>"""Conditional module is the xmodule, which you can use for disabling
some xmodules by conditions.
"""

import json
import logging
from lxml import etree
from pkg_resources import resource_string

from xmodule.x_module import XModule
from xmodule.modulestore import Location
from xmodule.seq_module import SequenceDescriptor
from xblock.core import String, Scope, List
from xmodule.modulestore.exceptions import ItemNotFoundError

log = logging.getLogger('mitx.' + __name__)

class ConditionalModule(XModule):
    """
    Blocks child module from showing unless certain conditions are met.

    Example:

        <conditional sources="i4x://.../problem_1; i4x://.../problem_2" completed="True">
            <show sources="i4x://.../test_6; i4x://.../Avi_resources"/>
            <video url_name="secret_video" />
        </conditional>

        TODO string comparison
            multiple answer for every poll

    """

    js = {'coffee': [resource_string(__name__, 'js/src/javascript_loader.coffee'),
                     resource_string(__name__, 'js/src/conditional/display.coffee'),
                     resource_string(__name__, 'js/src/collapsible.coffee'),

                    ]}

    js_module_name = "Conditional"
    css = {'scss': [resource_string(__name__, 'css/capa/display.scss')]}

    contents = String(scope=Scope.content)

    # Map
    # key: <tag attribute in xml>
    # value: <name of module attribute>
    conditions_map = {
        'poll_answer': 'poll_answer',  # poll_question attr
        'completed': 'is_completed',  # capa_problem attr
        'attempted': 'is_attempted',  # capa_problem attr
        'voted': 'voted'  # poll_question attr
    }

    def _get_condition(self):
        # Get first valid condition.
        for xml_attr, attr_name in self.conditions_map.iteritems():
            xml_value = self.descriptor.xml_attributes.get(xml_attr)
            if xml_value:
                return xml_value, attr_name
        raise Exception('Error in conditional module: unknown condition "%s"'
            % xml_attr)

    def is_condition_satisfied(self):
        self.required_modules = [self.system.get_module(descriptor.location) for
            descriptor in self.descriptor.get_required_module_descriptors()]

        xml_value, attr_name = self._get_condition()

        if xml_value and self.required_modules:
            for module in self.required_modules:
                if not hasattr(module, attr_name):
                    raise Exception('Error in conditional module: \
                    required module {module} has no {module_attr}'.format(
                        module=module, module_attr=attr_name))

                attr = getattr(module, attr_name)
                if callable(attr):
                    attr = attr()

                if xml_value != str(attr):
                    break
            else:
                return True
        return False

    def get_html(self):
        # Calculate html ids of dependencies
        self.required_html_ids = [descriptor.location.html_id() for
            descriptor in self.descriptor.get_required_module_descriptors()]

        return self.system.render_template('conditional_ajax.html', {
            'element_id': self.location.html_id(),
            'id': self.id,
            'ajax_url': self.system.ajax_url,
            'depends': ';'.join(self.required_html_ids)
        })

    def handle_ajax(self, dispatch, post):
        """This is called by courseware.moduleodule_render, to handle
        an AJAX call.
        """
        if not self.is_condition_satisfied():
            message = self.descriptor.xml_attributes.get('message')
            context = {'module': self,
                       'message': message}
            html = self.system.render_template('conditional_module.html',
                context)
            return json.dumps({'html': [html], 'passed': False,
                                'message': bool(message)})

        if self.contents is None:
            self.contents = [self.system.get_module(child_descriptor.location
                ).get_html()
                for child_descriptor in self.descriptor.get_children()]

        html = self.contents
        return json.dumps({'html': html, 'passed': True})

class ConditionalDescriptor(SequenceDescriptor):
    """Descriptor for conditional xmodule."""
    _tag_name = 'conditional'

    module_class = ConditionalModule

    filename_extension = "xml"

    stores_state = True
    has_score = False

    show_tag_list = List(help="Poll answers", scope=Scope.content)

    @staticmethod
    def parse_sources(xml_element, system, return_descriptor=False):
        """Parse xml_element 'sources' attr and:
        if return_descriptor=True - return list of descriptors
        if return_descriptor=False - return list of lcoations
        """
        result = []
        sources = xml_element.get('sources')
        if sources:
            locations = [location.strip() for location in sources.split(';')]
            for location in locations:
                # Check valid location url.
                if Location.is_valid(location):
                    try:
                        descriptor = system.load_item(location)
                        if return_descriptor:
                            result.append(descriptor)
                        else:
                            result.append(location)
                    except ItemNotFoundError:
                        log.exception("Invalid module by location.")
        return result

    def get_required_module_descriptors(self):
        """Returns a list of XModuleDescritpor instances upon
        which this module depends.
        """
        return ConditionalDescriptor.parse_sources(
            self.xml_attributes, self.system, True)

    @classmethod
    def definition_from_xml(cls, xml_object, system):
        children = []
        show_tag_list = []
        for child in xml_object:
            if child.tag == 'show':
                location = ConditionalDescriptor.parse_sources(
                    child, system)
                children.extend(location)
                show_tag_list.extend(location)
            else:
                try:
                    descriptor = system.process_xml(etree.tostring(child))
                    module_url = descriptor.location.url()
                    children.append(module_url)
                except:
                    log.exception("Unable to load child when parsing Conditional.")
        return {'show_tag_list': show_tag_list}, children

    def definition_to_xml(self, resource_fs):
        xml_object = etree.Element(self._tag_name)
        for child in self.get_children():
            location = str(child.location)
            if location in self.show_tag_list:
                show_str = '<{tag_name} sources="{sources}" />'.format(
                    tag_name='show', sources=location)
                xml_object.append(etree.fromstring(show_str))
            else:
                xml_object.append(
                    etree.fromstring(child.export_to_xml(resource_fs)))
        return xml_object
<commit_msg>get_module get descr insted of loc<commit_after>"""Conditional module is the xmodule, which you can use for disabling
some xmodules by conditions.
"""

import json
import logging
from lxml import etree
from pkg_resources import resource_string

from xmodule.x_module import XModule
from xmodule.modulestore import Location
from xmodule.seq_module import SequenceDescriptor
from xblock.core import String, Scope, List
from xmodule.modulestore.exceptions import ItemNotFoundError

log = logging.getLogger('mitx.' + __name__)

class ConditionalModule(XModule):
    """
    Blocks child module from showing unless certain conditions are met.

    Example:

        <conditional sources="i4x://.../problem_1; i4x://.../problem_2" completed="True">
            <show sources="i4x://.../test_6; i4x://.../Avi_resources"/>
            <video url_name="secret_video" />
        </conditional>

        TODO string comparison
            multiple answer for every poll

    """

    js = {'coffee': [resource_string(__name__, 'js/src/javascript_loader.coffee'),
                     resource_string(__name__, 'js/src/conditional/display.coffee'),
                     resource_string(__name__, 'js/src/collapsible.coffee'),

                    ]}

    js_module_name = "Conditional"
    css = {'scss': [resource_string(__name__, 'css/capa/display.scss')]}

    contents = String(scope=Scope.content)

    # Map
    # key: <tag attribute in xml>
    # value: <name of module attribute>
    conditions_map = {
        'poll_answer': 'poll_answer',  # poll_question attr
        'completed': 'is_completed',  # capa_problem attr
        'attempted': 'is_attempted',  # capa_problem attr
        'voted': 'voted'  # poll_question attr
    }

    def _get_condition(self):
        # Get first valid condition.
        for xml_attr, attr_name in self.conditions_map.iteritems():
            xml_value = self.descriptor.xml_attributes.get(xml_attr)
            if xml_value:
                return xml_value, attr_name
        raise Exception('Error in conditional module: unknown condition "%s"'
            % xml_attr)

    def is_condition_satisfied(self):
        self.required_modules = [self.system.get_module(descriptor) for
            descriptor in self.descriptor.get_required_module_descriptors()]

        xml_value, attr_name = self._get_condition()

        if xml_value and self.required_modules:
            for module in self.required_modules:
                if not hasattr(module, attr_name):
                    raise Exception('Error in conditional module: \
                    required module {module} has no {module_attr}'.format(
                        module=module, module_attr=attr_name))

                attr = getattr(module, attr_name)
                if callable(attr):
                    attr = attr()

                if xml_value != str(attr):
                    break
            else:
                return True
        return False

    def get_html(self):
        # Calculate html ids of dependencies
        self.required_html_ids = [descriptor.location.html_id() for
            descriptor in self.descriptor.get_required_module_descriptors()]

        return self.system.render_template('conditional_ajax.html', {
            'element_id': self.location.html_id(),
            'id': self.id,
            'ajax_url': self.system.ajax_url,
            'depends': ';'.join(self.required_html_ids)
        })

    def handle_ajax(self, dispatch, post):
        """This is called by courseware.moduleodule_render, to handle
        an AJAX call.
        """
        if not self.is_condition_satisfied():
            message = self.descriptor.xml_attributes.get('message')
            context = {'module': self,
                       'message': message}
            html = self.system.render_template('conditional_module.html',
                context)
            return json.dumps({'html': [html], 'passed': False,
                                'message': bool(message)})

        if self.contents is None:
            self.contents = [self.system.get_module(child_descriptor).get_html()
                    for child_descriptor in self.descriptor.get_children()]

        html = self.contents
        return json.dumps({'html': html, 'passed': True})

class ConditionalDescriptor(SequenceDescriptor):
    """Descriptor for conditional xmodule."""
    _tag_name = 'conditional'

    module_class = ConditionalModule

    filename_extension = "xml"

    stores_state = True
    has_score = False

    show_tag_list = List(help="Poll answers", scope=Scope.content)

    @staticmethod
    def parse_sources(xml_element, system, return_descriptor=False):
        """Parse xml_element 'sources' attr and:
        if return_descriptor=True - return list of descriptors
        if return_descriptor=False - return list of locations
        """
        result = []
        sources = xml_element.get('sources')
        if sources:
            locations = [location.strip() for location in sources.split(';')]
            for location in locations:
                # Check valid location url.
                if Location.is_valid(location):
                    try:
                        descriptor = system.load_item(location)
                        if return_descriptor:
                            result.append(descriptor)
                        else:
                            result.append(location)
                    except ItemNotFoundError:
                        log.exception("Invalid module by location.")
        return result

    def get_required_module_descriptors(self):
        """Returns a list of XModuleDescritpor instances upon
        which this module depends.
        """
        return ConditionalDescriptor.parse_sources(
            self.xml_attributes, self.system, True)

    @classmethod
    def definition_from_xml(cls, xml_object, system):
        children = []
        show_tag_list = []
        for child in xml_object:
            if child.tag == 'show':
                location = ConditionalDescriptor.parse_sources(
                    child, system)
                children.extend(location)
                show_tag_list.extend(location)
            else:
                try:
                    descriptor = system.process_xml(etree.tostring(child))
                    module_url = descriptor.location.url()
                    children.append(module_url)
                except:
                    log.exception("Unable to load child when parsing Conditional.")
        return {'show_tag_list': show_tag_list}, children

    def definition_to_xml(self, resource_fs):
        xml_object = etree.Element(self._tag_name)
        for child in self.get_children():
            location = str(child.location)
            if location in self.show_tag_list:
                show_str = '<{tag_name} sources="{sources}" />'.format(
                    tag_name='show', sources=location)
                xml_object.append(etree.fromstring(show_str))
            else:
                xml_object.append(
                    etree.fromstring(child.export_to_xml(resource_fs)))
        return xml_object
<|endoftext|>


<commit_before>from django.conf import settings
from django.core.management.base import BaseCommand
from zebra import MAXIMUM_STRIPE_CUSTOMER_LIST_SIZE
import stripe

CLEAR_CHUNK_SIZE = MAXIMUM_STRIPE_CUSTOMER_LIST_SIZE

class Command(BaseCommand):
    help = "Clear all test mode customers from your stripe account."
    __test__ = False

    def handle(self, *args, **options):
        verbosity = int(options.get('verbosity', 1))
        stripe.api_key = settings.STRIPE_SECRET
        customer_chunk = [0]

        if verbosity > 0:
            print "Clearing stripe test customers:"

        num_checked = 0
        while customer_chunk is not []:
            customer_chunk = stripe.Customer.all(count=CLEAR_CHUNK_SIZE, offset=num_checked).data

            if verbosity > 1:
                print "Processing records %s-%s" % (num_checked, num_checked+len(customer_chunk))
            for c in customer_chunk:
                if verbosity > 2:
                    print "Deleting %s..." % (c.description),

                if not c.livemode:
                    c.delete()

                    if verbosity > 2:
                        print "done"

            num_checked = num_checked + len(customer_chunk)

        if verbosity > 0:
            print "Finished clearing stripe test customers."
<commit_msg>whitespace<commit_after>from django.conf import settings
from django.core.management.base import BaseCommand
from zebra import MAXIMUM_STRIPE_CUSTOMER_LIST_SIZE
import stripe

CLEAR_CHUNK_SIZE = MAXIMUM_STRIPE_CUSTOMER_LIST_SIZE

class Command(BaseCommand):
    help = "Clear all test mode customers from your stripe account."
    __test__ = False

    def handle(self, *args, **options):
        verbosity = int(options.get('verbosity', 1))
        stripe.api_key = settings.STRIPE_SECRET
        customer_chunk = [0]

        if verbosity > 0:
            print "Clearing stripe test customers:"

        num_checked = 0
        while customer_chunk is not []:
            customer_chunk = stripe.Customer.all(count=CLEAR_CHUNK_SIZE, offset=num_checked).data

            if verbosity > 1:
                print "Processing records %s-%s" % (num_checked, num_checked+len(customer_chunk))

            for c in customer_chunk:
                if verbosity > 2:
                    print "Deleting %s..." % (c.description),

                if not c.livemode:
                    c.delete()

                    if verbosity > 2:
                        print "done"

            num_checked = num_checked + len(customer_chunk)

        if verbosity > 0:
            print "Finished clearing stripe test customers."
<|endoftext|>


<commit_before># This file is part of the GOsa framework.
#
#  http://gosa-project.org
#
# Copyright:
#  (C) 2016 GONICUS GmbH, Germany, http://www.gonicus.de
#
# See the LICENSE file in the project's top-level directory for details.

import logging
import datetime
import paho.mqtt.client as mqtt
from gosa.common.gjson import loads, dumps
from tornado.queues import Queue, QueueEmpty
from tornado import gen
from gosa.common import Environment
from gosa.common.components import JSONRPCException, PluginRegistry

client_counter = 0

class BaseClient(mqtt.Client):  # pragma: nocover
    _clients = []

    def __init__(self, client_id=None, clean_session=True, userdata=None, protocol=mqtt.MQTTv31):
        super(BaseClient, self).__init__(client_id=client_id, clean_session=clean_session, userdata=userdata, protocol=protocol)
        BaseClient._clients.append(self)

    def get_thread(self):
        return self._thread

    @classmethod
    def get_clients(cls):
        return cls._clients

    def get_client_id(self):
        return self._client_id

class MQTTClient(object):
    """
    The MQTTClient is responsible for the connection to a MQTT broker (Mosquitto)
    Usage example:
        >>> from gosa.common.components.mqtt_client import MQTTClient
        >>>
        >>> def my_callback(topic, message):
        >>>     print("Message: %s, Topic: %s" % (message, topic))
        >>>
        >>> client = MQTTClient('localhost')
        >>> client.connect('username', 'password')
        >>> client.add_subscription('/topic/im/interested/in', callback=my_callback)
        >>> client.publish('/topic/to/publish', 'my message')
    """
    __published_messages = {}
    __sync_message_queues = {}
    __sender_id = None
    __connection_retries = 3
    __connection_retry_delay = 3
    __retried = 0

    def __init__(self, host, port=1883, keepalive=60, use_ssl=None, ca_file=None, insecure=None, client_id_prefix=None):
        global client_counter
        self.env = Environment.getInstance()
        self.log = logging.getLogger(__name__)

        self.connected = False

        uuid = self.env.core_uuid if hasattr(self.env, "core_uuid") else self.env.uuid
        if client_id_prefix is not None:
            client_id = "%s-%s-%s-%s" % (self.env.mode, client_id_prefix, uuid, client_counter)
        else:
            client_id = "%s-%s-%s" % (self.env.mode, uuid, client_counter)

        self.client = BaseClient(client_id=client_id)
        client_counter += 1
        self.host = host
        self.port = port
        self.keepalive = keepalive

        # set the callbacks
        self.client.on_connect = self.__on_connect
        self.client.on_message = self.__on_message
        self.client.on_subscribe = self.__on_subscribe
        self.client.on_unsubscribe = self.__on_unsubscribe
        self.client.on_publish = self.__on_publish
        self.client.on_log = self.__on_log
        if insecure is None:
            insecure = self.env.config.getboolean('mqtt.insecure')

        use_ssl = use_ssl if use_ssl is not None else self.env.config.getboolean('mqtt.ssl')
        if use_ssl is False and not self.env.config.has('mqtt.ssl'):
            # fallback to http.ssl setting
            use_ssl = self.env.config.getboolean('http.ssl')

        if use_ssl is True:
            cert = ca_file if ca_file is not None else self.env.config.get('mqtt.ca_file', default=None)
            if cert is not None:
                self.client.tls_set(cert)
                self.client.tls_insecure_set(insecure)

        self.subscriptions = {}

    def __on_log(self, client, userdata, level, buf):
        self.log.debug("%s: MQTT-log message: %s" % (self.get_identifier(), buf))

    def authenticate(self, uuid, secret=None):
        """ Send credentials to the MQTT broker.
        Note: must be called before connecting """
        self.__sender_id = uuid
        self.client.username_pw_set(uuid, secret)

    def connect(self, uuid=None, secret=None):
        """
        Connect to the MQTT broker
        :param uuid: username (optional)
        :param secret: password (optional)
        """
        if uuid is not None:
            self.authenticate(uuid, secret)
        self.client.connect(self.host, port=self.port, keepalive=self.keepalive)
        self.client.loop_start()
        self.env.threads.append(self.client.get_thread())

    def reconnect(self):
        self.client.reconnect()

    def disconnect(self):
        """ disconnect from the MQTT broker """
        self.client.disconnect()
        self.client.loop_stop()
        self.__sender_id = None

    def add_subscription(self, topic, callback=None, qos=0, sync=False):
        """ subscribe to a topic """
        if topic not in self.subscriptions.keys():
            self.subscriptions[topic] = {
                'subscribed': False,
                'qos': qos,
                'callback': callback,
                'sync': sync
            }
        if self.connected is True:
            (res, mid) = self.client.subscribe(topic)
            self.log.debug("%s: subscribing to '%s' => mid: '%s' == '%s'" % (self.get_identifier(), topic, mid, res))
            self.subscriptions[topic]['mid'] = mid
            self.subscriptions[topic]['subscription_result'] = res

    def get_identifier(self):
        """
        generates an identifier by combining the broker host
        and the client_id. Used mainly as preix for log messages
        """
        return "%s:%s/%s" % (self.host, self.port, self.client.get_client_id())

    def set_subscription_callback(self, callback):
        """ set a general callback for all subscriptions """
        for topic in self.subscriptions:
            self.subscriptions[topic]['callback'] = callback

    def __on_subscribe(self, client, userdata, mid, granted_qos):
        self.log.debug("%s: on_subscribe client='%s', userdata='%s', mid='%s', granted_qos='%s'" % (self.get_identifier(), client, userdata, mid, granted_qos))
        for topic in self.subscriptions:
            if 'mid' in self.subscriptions[topic] and self.subscriptions[topic]['mid'] == mid:
                self.subscriptions[topic]['granted_qos'] = granted_qos
                self.subscriptions[topic]['subscribed'] = True

    def __on_unsubscribe(self, client, userdata, mid):
        self.log.debug("%s: on_unsubscribe client='%s', userdata='%s', mid='%s'" % (self.get_identifier(), client, userdata, mid))
        for topic in list(self.subscriptions):
            if 'mid' in self.subscriptions[topic] and self.subscriptions[topic]['mid'] == mid:
                del self.subscriptions[topic]

    def remove_subscription(self, topic):
        """ unsubscribe from the given topic """
        if topic in self.subscriptions:
            del self.subscriptions[topic]
            if self.connected is True:
                self.client.unsubscribe(topic)

    def clear_subscriptions(self):
        """ unsubscribe from all subscribed topics """
        self.client.unsubscribe(self.subscriptions.keys())

    def __on_connect(self, client, userdata, flags, rc):
        if rc == mqtt.CONNACK_ACCEPTED:
            # connection successful
            self.log.info("%s: MQTT successfully connected" % self.get_identifier())
            self.connected = True
            self.__retried = 0
            for topic in self.subscriptions.keys():
                (res, mid) = self.client.subscribe(topic)
                self.log.debug("%s: subscribing to '%s' => mid: '%s' == '%s'" % (self.get_identifier(), topic, mid, res))
                self.subscriptions[topic]['mid'] = mid
                self.subscriptions[topic]['subscription_result'] = res
        else:
            self.connected = False
            msg = mqtt.error_string(rc)
            self.log.error("%s: MQTT connection error: %s" % (self.get_identifier(), msg))
            self.__reconnect()

    @gen.coroutine
    def __reconnect(self):
        if self.__retried < self.__connection_retries:
            yield gen.sleep(self.__connection_retry_delay)
            if self.connected is False:
                self.__retried += 1
                self.log.debug("%s: Reconnecting retry %s to %s" % (self.get_identifier(), self.__retried, self.host))
                self.reconnect()

    def __on_message(self, client, userdata, message):
        self.log.debug("%s: __on_message client='%s', userdata='%s', message='%s'" % (self.get_identifier(), client, userdata, message))
        payload = loads(message.payload)
        if isinstance(payload, dict) and "content" in payload:
            content = payload["content"]
            if self.__sender_id is not None and 'sender_id' in payload and payload['sender_id'] == self.__sender_id:
                # skip own messages
                return
        else:
            content = payload

        subs = self.get_subscriptions(message.topic)
        for sub in subs:
            if sub['sync'] is True:
                self.log.debug("%s: incoming message for synced topic %s" % (self.get_identifier(), message.topic))
                self.__sync_message_queues[message.topic].put(content)
            if 'callback' in sub and sub['callback'] is not None:
                callback = sub['callback']
                callback(message.topic, content)
            elif sub['sync'] is not True:
                self.log.warning("Incoming message not processed by async subscription because of missing callback")
        if len(subs) == 0:
            self.log.warning("Incoming message for unhandled topic '%s'" % message.topic)

    @gen.coroutine
    def get_sync_response(self, topic, message, qos=0):
        """
        Sends a message to a client queue and waits and returns the response from the client.

        :param topic: Topic this message should be sent to / received from
        :param message: The message published on the request topic
        :param qos: QOS value
        :return: The clients response
        """
        # listen on the backend response topic
        listen_to_topic = "%s/response" % topic
        self.__sync_message_queues[listen_to_topic] = Queue()
        self.add_subscription(listen_to_topic, sync=True)
        self.publish("%s/request" % topic, message, qos)
        # send to the client topic
        try:
            response = yield self.__sync_message_queues[listen_to_topic].get(timeout=datetime.timedelta(seconds=10))
            self.__sync_message_queues[listen_to_topic].task_done()
            return response
        except QueueEmpty:
            raise JSONRPCException("Timeout while waiting for the clients response")
        finally:
            self.remove_subscription(listen_to_topic)

    @gen.coroutine
    def publish(self, topic, message, qos=0, retain=False, retried=0):
        """ Publish a message on the MQTT bus"""
        message = {
            "sender_id": self.__sender_id,
            "content": message
        }
        res, mid = self.client.publish(topic, payload=dumps(message), qos=qos, retain=retain)

        self.__published_messages[mid] = res
        if res == mqtt.MQTT_ERR_NO_CONN:
            self.log.error("%s: mqtt server not reachable, message could not be send to '%s'" % (self.get_identifier(), topic))

            if qos > 0 and retried < 3:
                # try again
                yield gen.sleep(0.1)
                self.publish(topic, message, qos=qos, retain=retain, retried=retried+1)

    def will_set(self, topic, message, qos=0, retain=False):
        """
        Set a Will to be sent to the broker. If the client disconnects without calling disconnect(),
        the broker will publish the message on its behalf.
        """
        payload = {
            "content": message,
            "sender_id": self.__sender_id
        }
        self.client.will_set(topic, dumps(payload), qos, retain)

    def __on_publish(self, client, userdata, mid):
        if mid in self.__published_messages:
            del self.__published_messages[mid]

    def get_subscriptions(self, topic):
        """
        Find the subscriptions that match the given topic
        :param topic: Topic to check
        :return: list of found subscriptions
        """
        return [self.subscriptions[t] for t in self.subscriptions if mqtt.topic_matches_sub(t, topic)]
<commit_msg>do not log the pings<commit_after># This file is part of the GOsa framework.
#
#  http://gosa-project.org
#
# Copyright:
#  (C) 2016 GONICUS GmbH, Germany, http://www.gonicus.de
#
# See the LICENSE file in the project's top-level directory for details.

import logging
import datetime
import paho.mqtt.client as mqtt
from gosa.common.gjson import loads, dumps
from tornado.queues import Queue, QueueEmpty
from tornado import gen
from gosa.common import Environment
from gosa.common.components import JSONRPCException, PluginRegistry

client_counter = 0

class BaseClient(mqtt.Client):  # pragma: nocover
    _clients = []

    def __init__(self, client_id=None, clean_session=True, userdata=None, protocol=mqtt.MQTTv31):
        super(BaseClient, self).__init__(client_id=client_id, clean_session=clean_session, userdata=userdata, protocol=protocol)
        BaseClient._clients.append(self)

    def get_thread(self):
        return self._thread

    @classmethod
    def get_clients(cls):
        return cls._clients

    def get_client_id(self):
        return self._client_id

class MQTTClient(object):
    """
    The MQTTClient is responsible for the connection to a MQTT broker (Mosquitto)
    Usage example:
        >>> from gosa.common.components.mqtt_client import MQTTClient
        >>>
        >>> def my_callback(topic, message):
        >>>     print("Message: %s, Topic: %s" % (message, topic))
        >>>
        >>> client = MQTTClient('localhost')
        >>> client.connect('username', 'password')
        >>> client.add_subscription('/topic/im/interested/in', callback=my_callback)
        >>> client.publish('/topic/to/publish', 'my message')
    """
    __published_messages = {}
    __sync_message_queues = {}
    __sender_id = None
    __connection_retries = 3
    __connection_retry_delay = 3
    __retried = 0

    def __init__(self, host, port=1883, keepalive=60, use_ssl=None, ca_file=None, insecure=None, client_id_prefix=None):
        global client_counter
        self.env = Environment.getInstance()
        self.log = logging.getLogger(__name__)

        self.connected = False

        uuid = self.env.core_uuid if hasattr(self.env, "core_uuid") else self.env.uuid
        if client_id_prefix is not None:
            client_id = "%s-%s-%s-%s" % (self.env.mode, client_id_prefix, uuid, client_counter)
        else:
            client_id = "%s-%s-%s" % (self.env.mode, uuid, client_counter)

        self.client = BaseClient(client_id=client_id)
        client_counter += 1
        self.host = host
        self.port = port
        self.keepalive = keepalive

        # set the callbacks
        self.client.on_connect = self.__on_connect
        self.client.on_message = self.__on_message
        self.client.on_subscribe = self.__on_subscribe
        self.client.on_unsubscribe = self.__on_unsubscribe
        self.client.on_publish = self.__on_publish
        self.client.on_log = self.__on_log
        if insecure is None:
            insecure = self.env.config.getboolean('mqtt.insecure')

        use_ssl = use_ssl if use_ssl is not None else self.env.config.getboolean('mqtt.ssl')
        if use_ssl is False and not self.env.config.has('mqtt.ssl'):
            # fallback to http.ssl setting
            use_ssl = self.env.config.getboolean('http.ssl')

        if use_ssl is True:
            cert = ca_file if ca_file is not None else self.env.config.get('mqtt.ca_file', default=None)
            if cert is not None:
                self.client.tls_set(cert)
                self.client.tls_insecure_set(insecure)

        self.subscriptions = {}

    def __on_log(self, client, userdata, level, buf):
        if "PING" not in buf:
            self.log.debug("%s: MQTT-log message: %s" % (self.get_identifier(), buf))

    def authenticate(self, uuid, secret=None):
        """ Send credentials to the MQTT broker.
        Note: must be called before connecting """
        self.__sender_id = uuid
        self.client.username_pw_set(uuid, secret)

    def connect(self, uuid=None, secret=None):
        """
        Connect to the MQTT broker
        :param uuid: username (optional)
        :param secret: password (optional)
        """
        if uuid is not None:
            self.authenticate(uuid, secret)
        self.client.connect(self.host, port=self.port, keepalive=self.keepalive)
        self.client.loop_start()
        self.env.threads.append(self.client.get_thread())

    def reconnect(self):
        self.client.reconnect()

    def disconnect(self):
        """ disconnect from the MQTT broker """
        self.client.disconnect()
        self.client.loop_stop()
        self.__sender_id = None

    def add_subscription(self, topic, callback=None, qos=0, sync=False):
        """ subscribe to a topic """
        if topic not in self.subscriptions.keys():
            self.subscriptions[topic] = {
                'subscribed': False,
                'qos': qos,
                'callback': callback,
                'sync': sync
            }
        if self.connected is True:
            (res, mid) = self.client.subscribe(topic)
            self.log.debug("%s: subscribing to '%s' => mid: '%s' == '%s'" % (self.get_identifier(), topic, mid, res))
            self.subscriptions[topic]['mid'] = mid
            self.subscriptions[topic]['subscription_result'] = res

    def get_identifier(self):
        """
        generates an identifier by combining the broker host
        and the client_id. Used mainly as preix for log messages
        """
        return "%s:%s/%s" % (self.host, self.port, self.client.get_client_id())

    def set_subscription_callback(self, callback):
        """ set a general callback for all subscriptions """
        for topic in self.subscriptions:
            self.subscriptions[topic]['callback'] = callback

    def __on_subscribe(self, client, userdata, mid, granted_qos):
        self.log.debug("%s: on_subscribe client='%s', userdata='%s', mid='%s', granted_qos='%s'" % (self.get_identifier(), client, userdata, mid, granted_qos))
        for topic in self.subscriptions:
            if 'mid' in self.subscriptions[topic] and self.subscriptions[topic]['mid'] == mid:
                self.subscriptions[topic]['granted_qos'] = granted_qos
                self.subscriptions[topic]['subscribed'] = True

    def __on_unsubscribe(self, client, userdata, mid):
        self.log.debug("%s: on_unsubscribe client='%s', userdata='%s', mid='%s'" % (self.get_identifier(), client, userdata, mid))
        for topic in list(self.subscriptions):
            if 'mid' in self.subscriptions[topic] and self.subscriptions[topic]['mid'] == mid:
                del self.subscriptions[topic]

    def remove_subscription(self, topic):
        """ unsubscribe from the given topic """
        if topic in self.subscriptions:
            del self.subscriptions[topic]
            if self.connected is True:
                self.client.unsubscribe(topic)

    def clear_subscriptions(self):
        """ unsubscribe from all subscribed topics """
        self.client.unsubscribe(self.subscriptions.keys())

    def __on_connect(self, client, userdata, flags, rc):
        if rc == mqtt.CONNACK_ACCEPTED:
            # connection successful
            self.log.info("%s: MQTT successfully connected" % self.get_identifier())
            self.connected = True
            self.__retried = 0
            for topic in self.subscriptions.keys():
                (res, mid) = self.client.subscribe(topic)
                self.log.debug("%s: subscribing to '%s' => mid: '%s' == '%s'" % (self.get_identifier(), topic, mid, res))
                self.subscriptions[topic]['mid'] = mid
                self.subscriptions[topic]['subscription_result'] = res
        else:
            self.connected = False
            msg = mqtt.error_string(rc)
            self.log.error("%s: MQTT connection error: %s" % (self.get_identifier(), msg))
            self.__reconnect()

    @gen.coroutine
    def __reconnect(self):
        if self.__retried < self.__connection_retries:
            yield gen.sleep(self.__connection_retry_delay)
            if self.connected is False:
                self.__retried += 1
                self.log.debug("%s: Reconnecting retry %s to %s" % (self.get_identifier(), self.__retried, self.host))
                self.reconnect()

    def __on_message(self, client, userdata, message):
        self.log.debug("%s: __on_message client='%s', userdata='%s', message='%s'" % (self.get_identifier(), client, userdata, message))
        payload = loads(message.payload)
        if isinstance(payload, dict) and "content" in payload:
            content = payload["content"]
            if self.__sender_id is not None and 'sender_id' in payload and payload['sender_id'] == self.__sender_id:
                # skip own messages
                return
        else:
            content = payload

        subs = self.get_subscriptions(message.topic)
        for sub in subs:
            if sub['sync'] is True:
                self.log.debug("%s: incoming message for synced topic %s" % (self.get_identifier(), message.topic))
                self.__sync_message_queues[message.topic].put(content)
            if 'callback' in sub and sub['callback'] is not None:
                callback = sub['callback']
                callback(message.topic, content)
            elif sub['sync'] is not True:
                self.log.warning("Incoming message not processed by async subscription because of missing callback")
        if len(subs) == 0:
            self.log.warning("Incoming message for unhandled topic '%s'" % message.topic)

    @gen.coroutine
    def get_sync_response(self, topic, message, qos=0):
        """
        Sends a message to a client queue and waits and returns the response from the client.

        :param topic: Topic this message should be sent to / received from
        :param message: The message published on the request topic
        :param qos: QOS value
        :return: The clients response
        """
        # listen on the backend response topic
        listen_to_topic = "%s/response" % topic
        self.__sync_message_queues[listen_to_topic] = Queue()
        self.add_subscription(listen_to_topic, sync=True)
        self.publish("%s/request" % topic, message, qos)
        # send to the client topic
        try:
            response = yield self.__sync_message_queues[listen_to_topic].get(timeout=datetime.timedelta(seconds=10))
            self.__sync_message_queues[listen_to_topic].task_done()
            return response
        except QueueEmpty:
            raise JSONRPCException("Timeout while waiting for the clients response")
        finally:
            self.remove_subscription(listen_to_topic)

    @gen.coroutine
    def publish(self, topic, message, qos=0, retain=False, retried=0):
        """ Publish a message on the MQTT bus"""
        message = {
            "sender_id": self.__sender_id,
            "content": message
        }
        res, mid = self.client.publish(topic, payload=dumps(message), qos=qos, retain=retain)

        self.__published_messages[mid] = res
        if res == mqtt.MQTT_ERR_NO_CONN:
            self.log.error("%s: mqtt server not reachable, message could not be send to '%s'" % (self.get_identifier(), topic))

            if qos > 0 and retried < 3:
                # try again
                yield gen.sleep(0.1)
                self.publish(topic, message, qos=qos, retain=retain, retried=retried+1)

    def will_set(self, topic, message, qos=0, retain=False):
        """
        Set a Will to be sent to the broker. If the client disconnects without calling disconnect(),
        the broker will publish the message on its behalf.
        """
        payload = {
            "content": message,
            "sender_id": self.__sender_id
        }
        self.client.will_set(topic, dumps(payload), qos, retain)

    def __on_publish(self, client, userdata, mid):
        if mid in self.__published_messages:
            del self.__published_messages[mid]

    def get_subscriptions(self, topic):
        """
        Find the subscriptions that match the given topic
        :param topic: Topic to check
        :return: list of found subscriptions
        """
        return [self.subscriptions[t] for t in self.subscriptions if mqtt.topic_matches_sub(t, topic)]
<|endoftext|>


<commit_before>import mathstepsWrapper as mathsteps
from utils.ResponseCreation import ControllerResponse
import random
from algebraTemplate import ProblemGenerator
import re
import utils.MentiiLogging as MentiiLogging

def getProblem(problemTemplate):
  '''
  Expects a problem template of the form:
  <template>|<*min range>|<*max range>|<*operator list>

  The template can be any math expression with $a, $b, $c, $d for any numerical value, $var for the variable 'x', $op for a random operator from the operator list.

  The values for the min range, max range, and operator list are optional but if one is provided all values have to be provided. The default values are: -20, 20, and + - *
  '''
  problem = 'Bad Problem'
  problemTokens = problemTemplate.split('|')
  if len(problemTokens) == 4:
    try:
      template = problemTokens[0]
      minVal = int(problemTokens[1])
      maxVal = int(problemTokens[2])
      opVals = problemTokens[3].split(' ')
      generator = ProblemGenerator(minVal=minVal, maxVal=maxVal, opVals=opVals)
      problem = generator.buildProblemFromTemplate(template)
    except Exception as e:
      MentiiLogging.getLogger().exception(e)
  elif len(problemTokens) == 1:
    template = problemTokens[0]
    generator = ProblemGenerator()
    problem = generator.buildProblemFromTemplate(template)
  else:
    MentiiLogging.getLogger().warning('Could not build problem from template: {0}'.format(problemTemplate))

  return problem

def dropTerm(step):
  '''
  Always drop the right most term
  '''
  regex = '(\+|-|\*|/){1}\s*\d+' #Will match any number after an operator
  termList = [m.start() for m in re.finditer(regex, step)]
  res = step
  if len(termList) > 0:
    lastTerm = termList[-1]
    badStep = step[0:lastTerm] #Get everything upto but not inlcuding the last term.
    #Check to make sure we didnt drop the coefficent of x
    xIndex = step.index('x')
    newXIndex = badStep.index('x')
    if step[xIndex-1] != badStep[newXIndex - 1 ]: #The coeffiecent of X was dropped :(
      badstep = step
    res = badStep

  return res

def swapSign(step):
  signs = [m.start() for m in re.finditer('(\+|-)', step)]
  res = step
  if len(signs) > 0:
    signToSwap = random.choice(signs)
    badStep = list(step)
    if step[signToSwap] == '+':
      badStep[signToSwap] = '-'
    else:
      badStep[signToSwap] = '+'
    res = ''.join(badStep)

  return res

def swapOperator(step):
  operators = [m.start() for m in re.finditer('(\+|-|\*|/)', step)]
  res = step
  if len(operators) > 0:
    signToSwap = random.choice(operators)
    badStep = list(step)
    if step[signToSwap] == '+':
      badStep[signToSwap] = '*'
    elif step[signToSwap] == '-':
      badStep[signToSwap] = '/'
    elif step[signToSwap] == '/':
      badStep[signToSwap] = '*'
    elif step[signToSwap] == '*':
      badStep[signToSwap] = '/'
    res = ''.join(badStep)

  return res

def swapNumbers(step):
  numbers = [(m.start(), m.end()) for m in re.finditer('\d+', step)]
  res = step
  if len(numbers) > 1:
    firstNum, secondNum = random.sample(numbers, 2)
    if firstNum[0] > secondNum[0]:
      firstNum, secondNum = secondNum, firstNum
    badStep = []
    #Get everything upto the first number's start
    badStep.append(step[0:firstNum[0]])
    #Add the second number
    badStep.append(step[secondNum[0]:secondNum[1]])
    #Get everything from the first number's end to the second number start
    badStep.append(step[firstNum[1]:secondNum[0]])
    #Add the first number
    badStep.append(step[firstNum[0]:firstNum[1]])
    #Get everything from the second number's end to the end
    badStep.append(step[secondNum[1]:])
    res = ''.join(badStep)

  return res

def modifyStep(step):
  badStep = step
  success = False
  modifications = [dropTerm, swapSign, swapOperator, swapNumbers]
  modfunc = random.choice(modifications)
  badStep = modfunc(badStep)
  success = badStep != step

  return (badStep, success)

def getProblemTree(problemTemplate):
  #When problem templates work problem could
  # be a problem template instead. If that's the case
  # we could get the problem from the template here
  response = ControllerResponse()
  numberOfFailurePoints = 2 #TODO: Replace this with the recommender system output
  problem = getProblem(problemTemplate)
  problemPath = mathsteps.getStepsForProblem(problem)
  if len(problemPath) <= 1:
    #We couldn't get a path for the problem
    response.addError('Problem Solve Error', 'Could not generate path for problem {0}'.format(problem))
  else:
    problemTree = generateTreeWithBadSteps(problemPath, numberOfFailurePoints)
    response.addToPayload('problemTree', problemTree)

  return response

def generateTreeWithBadSteps(problemSolutionPath, numOfFailurePoints, failurePoints=None):
  # if testing failure points can be passed in
  # randomly pick failure points from problem solution path
  if not failurePoints and numOfFailurePoints < (len(problemSolutionPath) - 1):
    #Use a slice so we don't pick the first step as a failure point
    failurePoints = random.sample(problemSolutionPath[1:-1], numOfFailurePoints)
  elif failurePoints:
    pass #Keep the failure points as they were passed in
  else:
    failurePoints = []

  response = []
  for correctStep  in problemSolutionPath:
    step = {}
    step['correctStep'] = correctStep
    response.append(step)

  # for each failure point generate the bad step and path
  for step in failurePoints:
    failureIndex = problemSolutionPath.index(step)
    badStep, successful = modifyStep(step)
    if successful:
      badStepPath = mathsteps.getStepsForProblem(badStep)
      response[failureIndex]['badStep'] = badStep
      response[failureIndex]['badStepPath'] = badStepPath

  return response
<commit_msg>Improving bad steps<commit_after>import mathstepsWrapper as mathsteps
from utils.ResponseCreation import ControllerResponse
import random
from algebraTemplate import ProblemGenerator
import re
import utils.MentiiLogging as MentiiLogging

def getProblem(problemTemplate):
  '''
  Expects a problem template of the form:
  <template>|<*min range>|<*max range>|<*operator list>

  The template can be any math expression with $a, $b, $c, $d for any numerical value, $var for the variable 'x', $op for a random operator from the operator list.

  The values for the min range, max range, and operator list are optional but if one is provided all values have to be provided. The default values are: -20, 20, and + - *
  '''
  problem = 'Bad Problem'
  problemTokens = problemTemplate.split('|')
  if len(problemTokens) == 4:
    try:
      template = problemTokens[0]
      minVal = int(problemTokens[1])
      maxVal = int(problemTokens[2])
      opVals = problemTokens[3].split(' ')
      generator = ProblemGenerator(minVal=minVal, maxVal=maxVal, opVals=opVals)
      problem = generator.buildProblemFromTemplate(template)
    except Exception as e:
      MentiiLogging.getLogger().exception(e)
  elif len(problemTokens) == 1:
    template = problemTokens[0]
    generator = ProblemGenerator()
    problem = generator.buildProblemFromTemplate(template)
  else:
    MentiiLogging.getLogger().warning('Could not build problem from template: {0}'.format(problemTemplate))

  return problem

def dropTerm(step):
  '''
  Always drop the right most term
  '''
  regex = '(\+|-|\*|/| )+\s*\d+' #Will match any number after an operator
  termList = [m.start() for m in re.finditer(regex, step)]
  res = step
  if len(termList) > 0:
    lastTerm = termList[-1]
    badStep = step[0:lastTerm] #Get everything upto but not inlcuding the last term.
    #Check to make sure we didnt drop the coefficent of x
    xIndex = step.index('x')
    newXIndex = badStep.index('x')
    if step[xIndex-1] != badStep[newXIndex - 1 ]: #The coeffiecent of X was dropped :(
      badstep = step
    res = badStep

    if not '=' in res or res[-1] == '=':
      res = step

  return res

def swapSign(step):
  signs = [m.start() for m in re.finditer('(\+|-)', step)]
  res = step
  if len(signs) > 0:
    signToSwap = random.choice(signs)
    badStep = list(step)
    if step[signToSwap] == '+':
      badStep[signToSwap] = '-'
    else:
      badStep[signToSwap] = '+'
    res = ''.join(badStep)

  return res

def swapOperator(step):
  operators = [m.start() for m in re.finditer('(\+|-|\*|/)', step)]
  res = step
  if len(operators) > 0:
    signToSwap = random.choice(operators)
    badStep = list(step)
    if step[signToSwap] == '+':
      badStep[signToSwap] = '*'
    elif step[signToSwap] == '-':
      badStep[signToSwap] = '/'
    elif step[signToSwap] == '/':
      badStep[signToSwap] = '*'
    elif step[signToSwap] == '*':
      badStep[signToSwap] = '/'
    res = ''.join(badStep)

    if step[signToSwap+1] == 'x':
      res = step

  return res

def swapNumbers(step):
  numbers = [(m.start(), m.end()) for m in re.finditer('-?\d+', step)]
  res = step
  if len(numbers) > 1:
    firstNum, secondNum = random.sample(numbers, 2)
    if firstNum[0] > secondNum[0]:
      firstNum, secondNum = secondNum, firstNum
    badStep = []
    #Get everything upto the first number's start
    badStep.append(step[0:firstNum[0]])
    #Add the second number
    badStep.append(step[secondNum[0]:secondNum[1]])
    #Get everything from the first number's end to the second number start
    badStep.append(step[firstNum[1]:secondNum[0]])
    #Add the first number
    badStep.append(step[firstNum[0]:firstNum[1]])
    #Get everything from the second number's end to the end
    badStep.append(step[secondNum[1]:])
    res = ''.join(badStep)

  return res

def newTerm(step):
  op = random.choice(["+", "-"])
  num = random.choice(xrange(-10, 10))
  term = " {0} {1}".format(op, num)
  res = step + term
  return res

def modifyStep(step):
  badStep = step
  success = False
  modifications = [dropTerm, swapSign, swapOperator, swapNumbers, newTerm]
  modfunc = random.choice(modifications)
  badStep = modfunc(badStep)
  success = badStep != step

  return (badStep, success)

def getProblemTree(problemTemplate):
  #When problem templates work problem could
  # be a problem template instead. If that's the case
  # we could get the problem from the template here
  response = ControllerResponse()
  numberOfFailurePoints = 2 #TODO: Replace this with the recommender system output
  problem = getProblem(problemTemplate)
  problemPath = mathsteps.getStepsForProblem(problem)
  if len(problemPath) <= 1:
    #We couldn't get a path for the problem
    response.addError('Problem Solve Error', 'Could not generate path for problem {0}'.format(problem))
  else:
    problemTree = generateTreeWithBadSteps(problemPath, numberOfFailurePoints)
    response.addToPayload('problemTree', problemTree)

  return response

def generateTreeWithBadSteps(problemSolutionPath, numOfFailurePoints, failurePoints=None):
  # if testing failure points can be passed in
  # randomly pick failure points from problem solution path
  if not failurePoints and numOfFailurePoints < (len(problemSolutionPath) - 1):
    #Use a slice so we don't pick the first step as a failure point
    failurePoints = random.sample(problemSolutionPath[1:-1], numOfFailurePoints)
  elif failurePoints:
    pass #Keep the failure points as they were passed in
  else:
    failurePoints = []

  response = []
  for correctStep  in problemSolutionPath:
    step = {}
    step['correctStep'] = correctStep
    response.append(step)

  # for each failure point generate the bad step and path
  for step in failurePoints:
    failureIndex = problemSolutionPath.index(step)
    badStep, successful = modifyStep(step)
    if successful:
      badStepPath = mathsteps.getStepsForProblem(badStep)
      response[failureIndex]['badStep'] = badStep
      response[failureIndex]['badStepPath'] = badStepPath

  return response
<|endoftext|>


<commit_before>from nose.tools import assert_true, assert_false
from forgetracker.tests import TestController
from pyforge import model

class TestFunctionalController(TestController):

    def test_new_ticket(self):
        response = self.app.get('/bugs/new/')
        form = response.form
        summary = 'test new ticket'
        form['summary'] = summary
        response = form.submit().follow()
        assert_true(summary in response)

    def test_two_trackers(self):
        response = self.app.get('/doc_bugs/new/')
        form = response.form
        summary = 'test two trackers'
        form['summary'] = summary
        response = form.submit().follow()
        assert_true(summary in response)
        response = self.app.get('/bugs/')
        assert_false(summary in response)

    def test_new_comment(self):
        response = self.app.get('/bugs/new/')
        form = response.form
        form['summary'] = 'test new comment'
        form.submit()
        comment = 'comment testing new comment'
        self.app.post('/bugs/1/comments/reply', { 'text': comment })
        response = self.app.get('/bugs/1/')
        assert_true(comment in response)

    def test_render_ticket(self):
        response = self.app.get('/bugs/new/')
        form = response.form
        summary = 'test render ticket'
        form['summary'] = summary
        response = form.submit().follow()
        assert_true(summary in response)
        assert_true('Attachments' in response)
        assert_true('Comments' in response)
        assert_true('Make a comment' in response)

    def test_render_index(self):
        response = self.app.get('/bugs/new/')
        form = response.form
        summary = 'test render index'
        form['summary'] = summary
        form.submit()
        response = self.app.get('/bugs/')
        assert_true(summary in response)
<commit_msg>Cleanup: factor ticket creation in Tracker tests<commit_after>from nose.tools import assert_true, assert_false
from forgetracker.tests import TestController
from pyforge import model

class TestFunctionalController(TestController):

    def new_ticket(self, summary, mp='/bugs/'):
        response = self.app.get(mp + 'new/')
        form = response.form
        form['summary'] = summary
        return form.submit().follow()

    def test_new_ticket(self):
        summary = 'test new ticket'
        response = self.new_ticket(summary)
        assert_true(summary in response)

    def test_two_trackers(self):
        summary = 'test two trackers'
        response = self.new_ticket(summary, '/doc_bugs/')
        assert_true(summary in response)
        response = self.app.get('/bugs/')
        assert_false(summary in response)

    def test_new_comment(self):
        self.new_ticket('test new comment')
        comment = 'comment testing new comment'
        self.app.post('/bugs/1/comments/reply', { 'text': comment })
        response = self.app.get('/bugs/1/')
        assert_true(comment in response)

    def test_render_ticket(self):
        summary = 'test render ticket'
        response = self.new_ticket(summary)
        assert_true(summary in response)
        assert_true('Attachments' in response)
        assert_true('Comments' in response)
        assert_true('Make a comment' in response)

    def test_render_index(self):
        summary = 'test render index'
        self.new_ticket(summary)
        response = self.app.get('/bugs/')
        assert_true(summary in response)
<|endoftext|>