<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
Utilities: Agave wrapper for sherlock
"""
import os
from os import path as op
from subprocess import check_output, CalledProcessError, STDOUT
import re
from time import sleep
import logging

import pkg_resources as pkgr
from cappat import AGAVE_JOB_LOGS
from cappat.tpl import Template
from cappat.utils import check_folder, getsystemname

SLURM_FAIL_STATUS = ['CA', 'F', 'TO', 'NF', 'SE']
SLURM_WAIT_STATUS = ['R', 'PD', 'CF', 'CG']
SLEEP_SECONDS = 5

JOB_LOG = logging.getLogger('taskmanager')

class TaskManager(object):
    """
    A task manager factory class
    """
    def __init__(self):
        raise RuntimeError('This class cannot be instatiated.')

    @staticmethod
    def build(task_list, slurm_settings=None, work_dir=None,
              hostname=None):
        """
        Get the appropriate TaskManager object
        """
        hostname = slurm_settings.get('execution_system', None)

        if hostname is None:
            hostname = getsystemname()

        JOB_LOG.info('Identified host: "%s"', hostname)

        if not hostname:
            raise RuntimeError('Could not identify execution system')

        if hostname.endswith('ls5.tacc.utexas.edu'):
            return Lonestar5Submission(task_list, slurm_settings, work_dir)
        elif hostname.endswith('stanford.edu'):
            return SherlockSubmission(task_list, slurm_settings, work_dir)
        elif hostname.endswith('stampede.tacc.utexas.edu'):
            raise NotImplementedError
        elif hostname == 'test.circleci':
            return CircleCISubmission(task_list, slurm_settings, work_dir)
        elif hostname == 'test.local':
            return TestSubmission(task_list, slurm_settings, work_dir)
        else:
            raise RuntimeError(
                'Could not identify "{}" as a valid execution system'.format(hostname))

class TaskSubmissionBase(object):
    """
    A base class for task submission
    """
    slurm_settings = {}
    jobexp = re.compile(r'Submitted batch job (?P<jobid>\d*)')
    _cmd_prefix = []

    SLURM_TEMPLATE = pkgr.resource_filename('cappat', 'tpl/sherlock-sbatch.jnj2')

    def __init__(self, task_list, slurm_settings=None, group_cmd=None, work_dir=None):

        if not task_list:
            raise RuntimeError('a list of tasks is required')

        self.task_list = task_list

        if slurm_settings is not None:
            self.slurm_settings.update(slurm_settings)

        if work_dir is None:
            work_dir = os.getcwd()

        self.slurm_settings['child_runtime'] = _secs2time(
            int(0.85 * _time2secs(self.slurm_settings['time'])))

        self.work_dir = check_folder(op.abspath(work_dir))
        self.aux_dir = check_folder(op.join(self.work_dir, AGAVE_JOB_LOGS))
        self.slurm_settings.update(
            {'work_dir': self.work_dir, 'aux_dir': self.aux_dir}
        )
        self.sbatch_files = self._generate_sbatch()
        self._jobs = {}
        self._group_cmd = group_cmd

        JOB_LOG.info('Created TaskManager type "%s"', self.__class__.__name__)

    @property
    def job_ids(self):
        return list(self._jobs.keys())

    @property
    def jobs(self):
        return self._jobs

    @property
    def group_cmd(self):
        return self._group_cmd

    def _parse_jobid(self, slurm_msg):
        if isinstance(slurm_msg, (list, tuple)):
            slurm_msg = '\n'.join(slurm_msg)

        jobid = self.jobexp.search(slurm_msg).group('jobid')
        if jobid:
            self._jobs[jobid] = 'SUBMITTED'
        else:
            raise RuntimeError('Job ID could not extracted. Slurm message:\n{}'.format(
                slurm_msg))
        return jobid

    def _generate_sbatch(self):
        raise NotImplementedError

    def _submit_sbatch(self, task):
        return _run_cmd(self._cmd_prefix + ['sbatch', task])

    def _run_sacct(self):
        # sacct -n -X -j 10016750,10016749 -o JobID,State,ExitCode
        return _run_cmd(self._cmd_prefix + [
            'sacct', '-n', '-X', '-j', ','.join(self.job_ids),
            '-o', 'JobID,State,ExitCode'])

    def _get_job_acct(self):
        JOB_LOG.info('Checking exit code of jobs %s', ' '.join(self.job_ids))
        results = self._run_sacct()

        if results is None:
            JOB_LOG.critical('Running sacct over jobs %s did not produce any output',
                             ', '.join(self.job_ids))
            raise RuntimeError('sacct command output is empty')

        #parse results
        exit_codes = []
        for line in results.split('\n'):
            fields = line.split()
            self._jobs[fields[0]] = fields[1]
            exit_codes.append(int(fields[2].split(':')[0]))
        return exit_codes

    def _get_jobs_status(self):
        statuses = _run_cmd(self._cmd_prefix + [
            'squeue', '-j', ','.join(self.job_ids), '-o', '%i,%t', '-h'])

        # Jobs are not in the queue anymore
        if statuses is None:
            return True

        statuses = statuses.split('\n')
        if statuses[0].endswith('Invalid job id specified'):
            return True

        pending = []
        for line in statuses:
            status = line.split(',')
            if len(status) < 2:
                raise RuntimeError('Error parsing squeue output: {}'.format(
                    line))

            self._jobs[status[0]] = status[1]
            pending.append(status[0])

            if status in SLURM_FAIL_STATUS:
                raise RuntimeError('Job id {} failed with status {}.'.format(
                                   *status))

        JOB_LOG.info('There are pending jobs: %s', ' '.join(pending))
        return False

    def map_participant(self):
        """
        Submits a list of sbatch files and returns the assigned job ids
        """
        for i, task in enumerate(self.sbatch_files):
            JOB_LOG.info('Submitting sbatch/launcher file %s (%d)', task, i)
            # run sbatch
            sresult = self._submit_sbatch(task)
            # parse output and get job id
            jobid = self._parse_jobid(sresult)
            JOB_LOG.info(
                'Submitted task %d, job ID %s was assigned', i, jobid)

    def wait_participant(self):
        """
        Busy wait until all jobs in the list are done
        """
        JOB_LOG.info('Starting busy wait on jobs %s',
                     ' '.join(self.job_ids))
        all_finished = False

        while not all_finished:
            all_finished = self._get_jobs_status()
            sleep(SLEEP_SECONDS)

        JOB_LOG.info('Finished wait on jobs %s',
                     ' '.join(self.job_ids))

        # Run sacct to check the exit code of jobs
        overall_exit = sum(self._get_job_acct())

        JOB_LOG.info('Final status of jobs: %s', ', '.join([
            '%s (%s)' % (k, v) for k, v in list(self._jobs.items())]))

        if overall_exit > 0:
            failed_jobs = ['%s (logfiles: log/bidsapp-%s.{err,out}).' % k for k, v in list(
                self._jobs.items()) if v != 'COMPLETED']
            JOB_LOG.critical('One or more tasks finished with non-zero code:\n%s\t',
                             '\n\t'.join(failed_jobs))
            raise RuntimeError('One or more tasks finished with non-zero code')
        return self.job_ids

    def run_grouplevel(self):
        """
        Run the reduce operation over the participant map
        """
        if self._group_cmd is None:
            JOB_LOG.warning('Group level command not set, skipping reduce operation.')
            return None

        JOB_LOG.info('Kicking off reduce operation')
        return _run_cmd(self._group_cmd)

class Lonestar5Submission(TaskSubmissionBase):
    """
    The LS5 submission manager
    """

    def _generate_sbatch(self):
        """
        Generates one launcher file
        """
        launcher_file = op.join(self.aux_dir, 'launch_script.sh')
        with open(launcher_file, 'w') as lfh:
            lfh.write('\n'.join(self.task_list) + '\n')
        return [launcher_file]

    def _submit_sbatch(self, task):
        with open(task, 'r') as tfh:
            nodes = sum(1 for line in tfh if line.strip() and not line.strip().startswith('#'))
        values = {
            'cwd': os.getcwd(),
            'launcher_file': task,
            'nodes': nodes,
            'ncpus': 1,
            'jobname': self.slurm_settings['job_name']
        }
        launcher_cmd = """\
export LAUNCHER_WORKDIR={cwd}; \
/corral-repl/utexas/poldracklab/users/wtriplet/external/ls5_launch/launch -s {launcher_file} \
-n {ncpus} -N {nodes} -d {cwd} -r {runtime} -j {jobname}\
""".format(**values)
        return _run_cmd(['ssh', '-oStrictHostKeyChecking=no', 'login2', launcher_cmd])

class SherlockSubmission(TaskSubmissionBase):
    """
    The Sherlock submission
    """
    slurm_settings = {
        'modules': ['load singularity'],
        'srun_cmd': 'srun'
    }

    def __init__(self, task_list, slurm_settings=None, work_dir=None):
        if not slurm_settings is None:
            self.slurm_settings.update(slurm_settings)
        self.slurm_settings['qos'] = self.slurm_settings['partition']
        super(SherlockSubmission, self).__init__(
            task_list, work_dir=work_dir)

    def _generate_sbatch(self):
        """
        Generates one sbatch file per task
        """
        slurm_settings = self.slurm_settings.copy()
        sbatch_files = []
        for i, task in enumerate(self.task_list):
            sbatch_files.append(op.join(self.aux_dir, 'slurm-%06d.sbatch' % i))
            slurm_settings['commandline'] = task
            conf = Template(self.SLURM_TEMPLATE)
            conf.generate_conf(slurm_settings, sbatch_files[-1])
        return sbatch_files

class CircleCISubmission(SherlockSubmission):
    """
    A CircleCI submission manager to work with the slurm docker image
    """
    slurm_settings = {
        'nodes': 1,
        'time': '01:00:00',
        'partition': 'debug',
        'job_name': 'crn-bidsapp'
    }
    _cmd_prefix = ['sshpass', '-p', 'testpass',
                   'ssh', '-p', '10022', 'circleci@localhost']

    def _generate_sbatch(self):
        """
        Generates one sbatch file per task
        """
        # Remove default settings of Sherlock which are unsupported
        self.slurm_settings.pop('qos', None)
        self.slurm_settings.pop('mincpus', None)
        self.slurm_settings.pop('mem_per_cpu', None)
        self.slurm_settings.pop('modules', None)
        self.slurm_settings['work_dir'] = self.slurm_settings['work_dir'].replace(
            op.expanduser('~/'), '/')
        self.slurm_settings['work_dir'] = self.slurm_settings['work_dir'].replace(
            '~/', '/')
        return super(CircleCISubmission, self)._generate_sbatch()

    def _submit_sbatch(self, task):
        # Fix paths for docker image in CircleCI
        task = task.replace(op.expanduser('~/'), '/')
        task = task.replace('~/', '/')
        return super(CircleCISubmission, self)._submit_sbatch(task)

class TestSubmission(SherlockSubmission):
    """
    A Test submission manager to work with the slurm docker image
    """
    def _generate_sbatch(self):
        """
        Generates one sbatch file per task
        """
        # Remove default settings of Sherlock not supported
        self.slurm_settings.pop('qos', None)
        self.slurm_settings.pop('mincpus', None)
        self.slurm_settings.pop('mem_per_cpu', None)
        self.slurm_settings.pop('modules', None)
        self.slurm_settings.pop('srun_cmd', None)
        return super(TestSubmission, self)._generate_sbatch()

    def _submit_sbatch(self, task):
        return _run_cmd(['/bin/bash', task])

    def _get_jobs_status(self):
        jobs = ['%s,COMPLETED' % j for j in self.job_ids]
        return _run_cmd(['echo', '\n'.join(jobs)]).strip()

    def _run_sacct(self):
        return '\n'.join(['%s  COMPLETED  0:0' % j for j in self.job_ids])

def _run_cmd(cmd, shell=False):
    JOB_LOG.info('Executing command line: %s', ' '.join(cmd))
    try:
        result = check_output(cmd, stderr=STDOUT, shell=shell)
    except CalledProcessError as error:
        JOB_LOG.critical('Error submitting (exit code %d): \n\tCmdline: %s\n\tOutput:\n\t%s',
                         error.returncode, ' '.join(cmd), error.output)
        raise
    result = '\n'.join([line for line in result.split('\n') if line.strip()])
    if not result:
        JOB_LOG.info('Command output was empty')
        return None

    JOB_LOG.info('Command output: \n%s', result)
    return result

def _time2secs(timestr):
    return sum((60**i) * int(t) for i, t in enumerate(reversed(timestr.split(':'))))

def _secs2time(seconds):
    m, s = divmod(seconds, 60)
    h, m = divmod(m, 60)
    return "%02d:%02d:%02d" % (h, m, s)

<commit_msg>fix max_runtime key<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
Utilities: Agave wrapper for sherlock
"""
import os
from os import path as op
from subprocess import check_output, CalledProcessError, STDOUT
import re
from time import sleep
import logging

import pkg_resources as pkgr
from cappat import AGAVE_JOB_LOGS
from cappat.tpl import Template
from cappat.utils import check_folder, getsystemname

SLURM_FAIL_STATUS = ['CA', 'F', 'TO', 'NF', 'SE']
SLURM_WAIT_STATUS = ['R', 'PD', 'CF', 'CG']
SLEEP_SECONDS = 5

JOB_LOG = logging.getLogger('taskmanager')

class TaskManager(object):
    """
    A task manager factory class
    """
    def __init__(self):
        raise RuntimeError('This class cannot be instatiated.')

    @staticmethod
    def build(task_list, slurm_settings=None, work_dir=None,
              hostname=None):
        """
        Get the appropriate TaskManager object
        """
        hostname = slurm_settings.get('execution_system', None)

        if hostname is None:
            hostname = getsystemname()

        JOB_LOG.info('Identified host: "%s"', hostname)

        if not hostname:
            raise RuntimeError('Could not identify execution system')

        if hostname.endswith('ls5.tacc.utexas.edu'):
            return Lonestar5Submission(task_list, slurm_settings, work_dir)
        elif hostname.endswith('stanford.edu'):
            return SherlockSubmission(task_list, slurm_settings, work_dir)
        elif hostname.endswith('stampede.tacc.utexas.edu'):
            raise NotImplementedError
        elif hostname == 'test.circleci':
            return CircleCISubmission(task_list, slurm_settings, work_dir)
        elif hostname == 'test.local':
            return TestSubmission(task_list, slurm_settings, work_dir)
        else:
            raise RuntimeError(
                'Could not identify "{}" as a valid execution system'.format(hostname))

class TaskSubmissionBase(object):
    """
    A base class for task submission
    """
    slurm_settings = {}
    jobexp = re.compile(r'Submitted batch job (?P<jobid>\d*)')
    _cmd_prefix = []

    SLURM_TEMPLATE = pkgr.resource_filename('cappat', 'tpl/sherlock-sbatch.jnj2')

    def __init__(self, task_list, slurm_settings=None, group_cmd=None, work_dir=None):

        if not task_list:
            raise RuntimeError('a list of tasks is required')

        self.task_list = task_list

        if slurm_settings is not None:
            self.slurm_settings.update(slurm_settings)

        if work_dir is None:
            work_dir = os.getcwd()

        self.slurm_settings['child_runtime'] = _secs2time(
            int(0.85 * _time2secs(self.slurm_settings['max_runtime'])))

        self.work_dir = check_folder(op.abspath(work_dir))
        self.aux_dir = check_folder(op.join(self.work_dir, AGAVE_JOB_LOGS))
        self.slurm_settings.update(
            {'work_dir': self.work_dir, 'aux_dir': self.aux_dir}
        )
        self.sbatch_files = self._generate_sbatch()
        self._jobs = {}
        self._group_cmd = group_cmd

        JOB_LOG.info('Created TaskManager type "%s"', self.__class__.__name__)

    @property
    def job_ids(self):
        return list(self._jobs.keys())

    @property
    def jobs(self):
        return self._jobs

    @property
    def group_cmd(self):
        return self._group_cmd

    def _parse_jobid(self, slurm_msg):
        if isinstance(slurm_msg, (list, tuple)):
            slurm_msg = '\n'.join(slurm_msg)

        jobid = self.jobexp.search(slurm_msg).group('jobid')
        if jobid:
            self._jobs[jobid] = 'SUBMITTED'
        else:
            raise RuntimeError('Job ID could not extracted. Slurm message:\n{}'.format(
                slurm_msg))
        return jobid

    def _generate_sbatch(self):
        raise NotImplementedError

    def _submit_sbatch(self, task):
        return _run_cmd(self._cmd_prefix + ['sbatch', task])

    def _run_sacct(self):
        # sacct -n -X -j 10016750,10016749 -o JobID,State,ExitCode
        return _run_cmd(self._cmd_prefix + [
            'sacct', '-n', '-X', '-j', ','.join(self.job_ids),
            '-o', 'JobID,State,ExitCode'])

    def _get_job_acct(self):
        JOB_LOG.info('Checking exit code of jobs %s', ' '.join(self.job_ids))
        results = self._run_sacct()

        if results is None:
            JOB_LOG.critical('Running sacct over jobs %s did not produce any output',
                             ', '.join(self.job_ids))
            raise RuntimeError('sacct command output is empty')

        #parse results
        exit_codes = []
        for line in results.split('\n'):
            fields = line.split()
            self._jobs[fields[0]] = fields[1]
            exit_codes.append(int(fields[2].split(':')[0]))
        return exit_codes

    def _get_jobs_status(self):
        statuses = _run_cmd(self._cmd_prefix + [
            'squeue', '-j', ','.join(self.job_ids), '-o', '%i,%t', '-h'])

        # Jobs are not in the queue anymore
        if statuses is None:
            return True

        statuses = statuses.split('\n')
        if statuses[0].endswith('Invalid job id specified'):
            return True

        pending = []
        for line in statuses:
            status = line.split(',')
            if len(status) < 2:
                raise RuntimeError('Error parsing squeue output: {}'.format(
                    line))

            self._jobs[status[0]] = status[1]
            pending.append(status[0])

            if status in SLURM_FAIL_STATUS:
                raise RuntimeError('Job id {} failed with status {}.'.format(
                                   *status))

        JOB_LOG.info('There are pending jobs: %s', ' '.join(pending))
        return False

    def map_participant(self):
        """
        Submits a list of sbatch files and returns the assigned job ids
        """
        for i, task in enumerate(self.sbatch_files):
            JOB_LOG.info('Submitting sbatch/launcher file %s (%d)', task, i)
            # run sbatch
            sresult = self._submit_sbatch(task)
            # parse output and get job id
            jobid = self._parse_jobid(sresult)
            JOB_LOG.info(
                'Submitted task %d, job ID %s was assigned', i, jobid)

    def wait_participant(self):
        """
        Busy wait until all jobs in the list are done
        """
        JOB_LOG.info('Starting busy wait on jobs %s',
                     ' '.join(self.job_ids))
        all_finished = False

        while not all_finished:
            all_finished = self._get_jobs_status()
            sleep(SLEEP_SECONDS)

        JOB_LOG.info('Finished wait on jobs %s',
                     ' '.join(self.job_ids))

        # Run sacct to check the exit code of jobs
        overall_exit = sum(self._get_job_acct())

        JOB_LOG.info('Final status of jobs: %s', ', '.join([
            '%s (%s)' % (k, v) for k, v in list(self._jobs.items())]))

        if overall_exit > 0:
            failed_jobs = ['%s (logfiles: log/bidsapp-%s.{err,out}).' % k for k, v in list(
                self._jobs.items()) if v != 'COMPLETED']
            JOB_LOG.critical('One or more tasks finished with non-zero code:\n%s\t',
                             '\n\t'.join(failed_jobs))
            raise RuntimeError('One or more tasks finished with non-zero code')
        return self.job_ids

    def run_grouplevel(self):
        """
        Run the reduce operation over the participant map
        """
        if self._group_cmd is None:
            JOB_LOG.warning('Group level command not set, skipping reduce operation.')
            return None

        JOB_LOG.info('Kicking off reduce operation')
        return _run_cmd(self._group_cmd)

class Lonestar5Submission(TaskSubmissionBase):
    """
    The LS5 submission manager
    """

    def _generate_sbatch(self):
        """
        Generates one launcher file
        """
        launcher_file = op.join(self.aux_dir, 'launch_script.sh')
        with open(launcher_file, 'w') as lfh:
            lfh.write('\n'.join(self.task_list) + '\n')
        return [launcher_file]

    def _submit_sbatch(self, task):
        with open(task, 'r') as tfh:
            nodes = sum(1 for line in tfh if line.strip() and not line.strip().startswith('#'))
        values = {
            'cwd': os.getcwd(),
            'launcher_file': task,
            'nodes': nodes,
            'ncpus': 1,
            'jobname': self.slurm_settings['job_name']
        }
        launcher_cmd = """\
export LAUNCHER_WORKDIR={cwd}; \
/corral-repl/utexas/poldracklab/users/wtriplet/external/ls5_launch/launch -s {launcher_file} \
-n {ncpus} -N {nodes} -d {cwd} -r {runtime} -j {jobname}\
""".format(**values)
        return _run_cmd(['ssh', '-oStrictHostKeyChecking=no', 'login2', launcher_cmd])

class SherlockSubmission(TaskSubmissionBase):
    """
    The Sherlock submission
    """
    slurm_settings = {
        'modules': ['load singularity'],
        'srun_cmd': 'srun'
    }

    def __init__(self, task_list, slurm_settings=None, work_dir=None):
        if not slurm_settings is None:
            self.slurm_settings.update(slurm_settings)
        self.slurm_settings['qos'] = self.slurm_settings['partition']
        super(SherlockSubmission, self).__init__(
            task_list, work_dir=work_dir)

    def _generate_sbatch(self):
        """
        Generates one sbatch file per task
        """
        slurm_settings = self.slurm_settings.copy()
        sbatch_files = []
        for i, task in enumerate(self.task_list):
            sbatch_files.append(op.join(self.aux_dir, 'slurm-%06d.sbatch' % i))
            slurm_settings['commandline'] = task
            conf = Template(self.SLURM_TEMPLATE)
            conf.generate_conf(slurm_settings, sbatch_files[-1])
        return sbatch_files

class CircleCISubmission(SherlockSubmission):
    """
    A CircleCI submission manager to work with the slurm docker image
    """
    slurm_settings = {
        'nodes': 1,
        'time': '01:00:00',
        'partition': 'debug',
        'job_name': 'crn-bidsapp'
    }
    _cmd_prefix = ['sshpass', '-p', 'testpass',
                   'ssh', '-p', '10022', 'circleci@localhost']

    def _generate_sbatch(self):
        """
        Generates one sbatch file per task
        """
        # Remove default settings of Sherlock which are unsupported
        self.slurm_settings.pop('qos', None)
        self.slurm_settings.pop('mincpus', None)
        self.slurm_settings.pop('mem_per_cpu', None)
        self.slurm_settings.pop('modules', None)
        self.slurm_settings['work_dir'] = self.slurm_settings['work_dir'].replace(
            op.expanduser('~/'), '/')
        self.slurm_settings['work_dir'] = self.slurm_settings['work_dir'].replace(
            '~/', '/')
        return super(CircleCISubmission, self)._generate_sbatch()

    def _submit_sbatch(self, task):
        # Fix paths for docker image in CircleCI
        task = task.replace(op.expanduser('~/'), '/')
        task = task.replace('~/', '/')
        return super(CircleCISubmission, self)._submit_sbatch(task)

class TestSubmission(SherlockSubmission):
    """
    A Test submission manager to work with the slurm docker image
    """
    def _generate_sbatch(self):
        """
        Generates one sbatch file per task
        """
        # Remove default settings of Sherlock not supported
        self.slurm_settings.pop('qos', None)
        self.slurm_settings.pop('mincpus', None)
        self.slurm_settings.pop('mem_per_cpu', None)
        self.slurm_settings.pop('modules', None)
        self.slurm_settings.pop('srun_cmd', None)
        return super(TestSubmission, self)._generate_sbatch()

    def _submit_sbatch(self, task):
        return _run_cmd(['/bin/bash', task])

    def _get_jobs_status(self):
        jobs = ['%s,COMPLETED' % j for j in self.job_ids]
        return _run_cmd(['echo', '\n'.join(jobs)]).strip()

    def _run_sacct(self):
        return '\n'.join(['%s  COMPLETED  0:0' % j for j in self.job_ids])

def _run_cmd(cmd, shell=False):
    JOB_LOG.info('Executing command line: %s', ' '.join(cmd))
    try:
        result = check_output(cmd, stderr=STDOUT, shell=shell)
    except CalledProcessError as error:
        JOB_LOG.critical('Error submitting (exit code %d): \n\tCmdline: %s\n\tOutput:\n\t%s',
                         error.returncode, ' '.join(cmd), error.output)
        raise
    result = '\n'.join([line for line in result.split('\n') if line.strip()])
    if not result:
        JOB_LOG.info('Command output was empty')
        return None

    JOB_LOG.info('Command output: \n%s', result)
    return result

def _time2secs(timestr):
    return sum((60**i) * int(t) for i, t in enumerate(reversed(timestr.split(':'))))

def _secs2time(seconds):
    m, s = divmod(seconds, 60)
    h, m = divmod(m, 60)
    return "%02d:%02d:%02d" % (h, m, s)

<|endoftext|>


<commit_before># © 2019 Eficent Business and IT Consulting Services S.L.
# © 2019 Serpent Consulting Services Pvt. Ltd.
# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
{
    "name": 'Accounting with Operating Units',
    "summary": "Introduces Operating Unit (OU) in invoices and "
               "Accounting Entries with clearing account",
    "version": "12.0.1.0.1",
    "author": "Eficent, "
              "Serpent Consulting Services Pvt. Ltd.,"
              "WilldooIT Pty Ltd,"
              "Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/operating-unit",
    "category": "Accounting & Finance",
    "depends": ['account', 'operating_unit', 'analytic_operating_unit'],
    "license": "LGPL-3",
    "data": [
        "security/account_security.xml",
        "views/account_move_view.xml",
        "views/account_journal_view.xml",
        "views/company_view.xml",
        "views/invoice_view.xml",
        "views/account_payment_view.xml",
        "views/account_invoice_report_view.xml",
    ],
}
<commit_msg>account_operating_unit 12.0.1.0.2<commit_after># © 2019 Eficent Business and IT Consulting Services S.L.
# © 2019 Serpent Consulting Services Pvt. Ltd.
# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
{
    "name": 'Accounting with Operating Units',
    "summary": "Introduces Operating Unit (OU) in invoices and "
               "Accounting Entries with clearing account",
    "version": "12.0.1.0.2",
    "author": "Eficent, "
              "Serpent Consulting Services Pvt. Ltd.,"
              "WilldooIT Pty Ltd,"
              "Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/operating-unit",
    "category": "Accounting & Finance",
    "depends": ['account', 'operating_unit', 'analytic_operating_unit'],
    "license": "LGPL-3",
    "data": [
        "security/account_security.xml",
        "views/account_move_view.xml",
        "views/account_journal_view.xml",
        "views/company_view.xml",
        "views/invoice_view.xml",
        "views/account_payment_view.xml",
        "views/account_invoice_report_view.xml",
    ],
}
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

import codecs
import grp
import json
import logging
import os
import pwd

import click
import jinja2.sandbox

import credsmash.api
from .util import read_many, shell_quote, parse_manifest, detect_format
from .cli import main

logger = logging.getLogger(__name__)

class CachingProxy(object):
    def __init__(self, getter, key_fmt):
        self._getter = getter
        self._key_fmt = key_fmt
        self._data = {}

    def __getitem__(self, key):
        if key not in self._data:
            if isinstance(key, tuple):
                lookup_key = self._key_fmt.format(*key)
            else:
                lookup_key = self._key_fmt.format(key)
            logger.debug('key=%s lookup_key=%s', key, lookup_key)
            res = self._getter(lookup_key)
            self._data[key] = res
            return res
        return self._data[key]

@main.command('render-template')
@click.argument('template', type=click.File(mode='r', encoding='utf-8'))
@click.argument('destination', type=click.File(mode='w', encoding='utf-8'))
@click.option('--obj-name', default='secrets',
              help='The variable/object name provided to the template')
@click.option('--key-fmt', default='{0}',
              help='Re-use templates by tweaking which variable it maps to- '
                   'eg, "dev.{0}" converts {{secrets.potato}} to the secret "dev.potato"')
@click.option('--template-vars', type=click.File(mode='rb'))
@click.option('--template-vars-format', default=None)
@click.option('--secrets-file', type=click.File(mode='rb'),
              help="Source from a local file instead of credential store "
                   "(useful for caching/testing)")
@click.option('--secrets-file-format', default=None)
@click.pass_context
def cmd_render_template(
        ctx, template, destination,
        obj_name='secrets', key_fmt='{0}',
        template_vars=None, template_vars_format=None,
        secrets_file=None, secrets_file_format=None
):
    """
    Render a configuration template....
    """
    if secrets_file:
        if not secrets_file_format:
            secrets_file_format = detect_format(secrets_file, 'json')
        local_secrets = read_many(secrets_file, secrets_file_format)
        secrets = CachingProxy(lambda key: local_secrets[key], key_fmt)
    else:
        secrets = CachingProxy(lambda key: credsmash.api.get_secret(
            ctx.obj.storage_service,
            ctx.obj.key_service,
            key,
        ), key_fmt)

    render_args = {}
    if template_vars:
        if not template_vars_format:
            template_vars_format = detect_format(template_vars, 'json')
        render_args = read_many(template_vars, template_vars_format)
    if obj_name in render_args:
        logger.warning('Overwrote %r from template vars with secrets var.', obj_name)
    render_args[obj_name] = secrets

    env = _make_env()
    output = env.from_string(template.read()).render(render_args)
    destination.write(output)

@main.command('render-templates')
@click.argument('manifest', type=click.File(mode='r', encoding='utf-8'))
@click.option('--manifest-format', default=None)
@click.option('--obj-name', default='secrets',
              help='The variable/object name provided to the template')
@click.option('--key-fmt', default='{0}',
              help='Re-use templates by tweaking which variable it maps to- '
                   'eg, "dev.{0}" converts {{secrets.potato}} to the secret "dev.potato"')
@click.option('--template-vars', type=click.File(mode='rb'))
@click.option('--template-vars-format', default=None)
@click.option('--secrets-file', type=click.File(mode='rb'),
              help="Source from a local file instead of credential store "
                   "(useful for caching/testing)")
@click.option('--secrets-file-format', default=None)
@click.pass_context
def cmd_render_template(
        ctx, manifest, manifest_format=None,
        obj_name='secrets', key_fmt='{0}',
        template_vars=None, template_vars_format=None,
        secrets_file=None, secrets_file_format=None
):
    """
    Render multiple configuration templates - reads from a manifest file.
    """
    if secrets_file:
        if not secrets_file_format:
            secrets_file_format = detect_format(secrets_file, 'json')
        local_secrets = read_many(secrets_file, secrets_file_format)
        secrets = CachingProxy(lambda key: local_secrets[key], key_fmt)
    else:
        secrets = CachingProxy(lambda key: credsmash.api.get_secret(
            ctx.obj.storage_service,
            ctx.obj.key_service,
            key,
        ), key_fmt)

    render_args = {}
    if template_vars:
        if not template_vars_format:
            template_vars_format = detect_format(template_vars, 'json')
        render_args = read_many(template_vars, template_vars_format)
    if obj_name in render_args:
        logger.warning('Overwrote %r from template vars with secrets var.', obj_name)
    render_args[obj_name] = secrets

    env = _make_env()
    if not manifest_format:
        manifest_format = detect_format(manifest, 'json')
    for entry in parse_manifest(manifest, manifest_format):
        if 'source' in entry:
            with codecs.open(entry['source'], 'r', encoding='utf-8') as template:
                output = env.from_string(template.read()).render(render_args)
            # Only open the file after rendering the template
            #  as we truncate the file when opening.
            with codecs.open(entry['destination'], 'w', encoding='utf-8') as destination:
                destination.write(output)
            logger.info('Rendered template="%s" destination="%s"', entry['source'], entry['destination'])
        elif 'secret' in entry:
            output = secrets[entry['secret']]
            with open(entry['destination'], 'wb') as destination:
                destination.write(output)
            logger.info('Wrote secret="%s" destination="%s"', entry['secret'], entry['destination'])
        else:
            raise RuntimeError('Manifest entry must contain a secret or source')

        if 'mode' in entry:
            os.chmod(
                entry['destination'],
                entry['mode']
            )

        if 'owner' in entry and 'group' in entry:
            os.chown(
                entry['destination'],
                pwd.getpwnam(entry['owner']).pw_uid,
                grp.getgrnam(entry['group']).gr_gid
            )

def _make_env():
    env = jinja2.sandbox.SandboxedEnvironment(
        trim_blocks=True,
        lstrip_blocks=True,
        autoescape=False,
    )
    env.filters['sh'] = shell_quote
    env.filters['jsonify'] = json.dumps
    return env
<commit_msg>Templates - Add support for `in` operator.<commit_after>from __future__ import absolute_import, division, print_function

import codecs
import grp
import json
import logging
import os
import pwd

import click
import jinja2.sandbox

import credsmash.api
from .util import read_many, shell_quote, parse_manifest, detect_format, ItemNotFound
from .cli import main

logger = logging.getLogger(__name__)

class CredsmashProxy(object):
    def __init__(self, key_service, storage_service, key_fmt):
        self._key_service = key_service
        self._storage_service = storage_service
        self._key_fmt = key_fmt
        self._data = {}

    def __getitem__(self, key):
        if key in self._data:
            return self._data[key]

        if isinstance(key, tuple):
            lookup_key = self._key_fmt.format(*key)
        else:
            lookup_key = self._key_fmt.format(key)
        logger.debug('key=%s lookup_key=%s', key, lookup_key)
        res = credsmash.api.get_secret(
            self._storage_service,
            self._key_service,
            key,
        )
        self._data[key] = res
        return res

    def __contains__(self, key):
        try:
            self.__getitem__(key)
            return True
        except ItemNotFound:
            return False

class DictProxy(object):
    def __init__(self, items, key_fmt):
        self._items = items
        self._key_fmt = key_fmt

    def __getitem__(self, key):
        if isinstance(key, tuple):
            lookup_key = self._key_fmt.format(*key)
        else:
            lookup_key = self._key_fmt.format(key)
        return self._items[lookup_key]

    def __contains__(self, key):
        try:
            self.__getitem__(key)
            return True
        except KeyError:
            return False

@main.command('render-template')
@click.argument('template', type=click.File(mode='r', encoding='utf-8'))
@click.argument('destination', type=click.File(mode='w', encoding='utf-8'))
@click.option('--obj-name', default='secrets',
              help='The variable/object name provided to the template')
@click.option('--key-fmt', default='{0}',
              help='Re-use templates by tweaking which variable it maps to- '
                   'eg, "dev.{0}" converts {{secrets.potato}} to the secret "dev.potato"')
@click.option('--template-vars', type=click.File(mode='rb'))
@click.option('--template-vars-format', default=None)
@click.option('--secrets-file', type=click.File(mode='rb'),
              help="Source from a local file instead of credential store "
                   "(useful for caching/testing)")
@click.option('--secrets-file-format', default=None)
@click.pass_context
def cmd_render_template(
        ctx, template, destination,
        obj_name='secrets', key_fmt='{0}',
        template_vars=None, template_vars_format=None,
        secrets_file=None, secrets_file_format=None
):
    """
    Render a configuration template....
    """
    if secrets_file:
        if not secrets_file_format:
            secrets_file_format = detect_format(secrets_file, 'json')
        local_secrets = read_many(secrets_file, secrets_file_format)
        secrets = DictProxy(local_secrets, key_fmt)
    else:
        secrets = CredsmashProxy(
            ctx.obj.storage_service,
            ctx.obj.key_service,
            key_fmt,
        )

    render_args = {}
    if template_vars:
        if not template_vars_format:
            template_vars_format = detect_format(template_vars, 'json')
        render_args = read_many(template_vars, template_vars_format)
    if obj_name in render_args:
        logger.warning('Overwrote %r from template vars with secrets var.', obj_name)
    render_args[obj_name] = secrets

    env = _make_env()
    output = env.from_string(template.read()).render(render_args)
    destination.write(output)

@main.command('render-templates')
@click.argument('manifest', type=click.File(mode='r', encoding='utf-8'))
@click.option('--manifest-format', default=None)
@click.option('--obj-name', default='secrets',
              help='The variable/object name provided to the template')
@click.option('--key-fmt', default='{0}',
              help='Re-use templates by tweaking which variable it maps to- '
                   'eg, "dev.{0}" converts {{secrets.potato}} to the secret "dev.potato"')
@click.option('--template-vars', type=click.File(mode='rb'))
@click.option('--template-vars-format', default=None)
@click.option('--secrets-file', type=click.File(mode='rb'),
              help="Source from a local file instead of credential store "
                   "(useful for caching/testing)")
@click.option('--secrets-file-format', default=None)
@click.pass_context
def cmd_render_template(
        ctx, manifest, manifest_format=None,
        obj_name='secrets', key_fmt='{0}',
        template_vars=None, template_vars_format=None,
        secrets_file=None, secrets_file_format=None
):
    """
    Render multiple configuration templates - reads from a manifest file.
    """
    if secrets_file:
        if not secrets_file_format:
            secrets_file_format = detect_format(secrets_file, 'json')
        local_secrets = read_many(secrets_file, secrets_file_format)
        secrets = DictProxy(local_secrets, key_fmt)
    else:
        secrets = CredsmashProxy(
            ctx.obj.key_service,
            ctx.obj.storage_service,
            key_fmt,
        )

    render_args = {}
    if template_vars:
        if not template_vars_format:
            template_vars_format = detect_format(template_vars, 'json')
        render_args = read_many(template_vars, template_vars_format)
    if obj_name in render_args:
        logger.warning('Overwrote %r from template vars with secrets var.', obj_name)
    render_args[obj_name] = secrets

    env = _make_env()
    if not manifest_format:
        manifest_format = detect_format(manifest, 'json')
    for entry in parse_manifest(manifest, manifest_format):
        if 'source' in entry:
            with codecs.open(entry['source'], 'r', encoding='utf-8') as template:
                output = env.from_string(template.read()).render(render_args)
            # Only open the file after rendering the template
            #  as we truncate the file when opening.
            with codecs.open(entry['destination'], 'w', encoding='utf-8') as destination:
                destination.write(output)
            logger.info('Rendered template="%s" destination="%s"', entry['source'], entry['destination'])
        elif 'secret' in entry:
            output = secrets[entry['secret']]
            with open(entry['destination'], 'wb') as destination:
                destination.write(output)
            logger.info('Wrote secret="%s" destination="%s"', entry['secret'], entry['destination'])
        else:
            raise RuntimeError('Manifest entry must contain a secret or source')

        if 'mode' in entry:
            os.chmod(
                entry['destination'],
                entry['mode']
            )

        if 'owner' in entry and 'group' in entry:
            os.chown(
                entry['destination'],
                pwd.getpwnam(entry['owner']).pw_uid,
                grp.getgrnam(entry['group']).gr_gid
            )

def _make_env():
    env = jinja2.sandbox.SandboxedEnvironment(
        trim_blocks=True,
        lstrip_blocks=True,
        autoescape=False,
    )
    env.filters['sh'] = shell_quote
    env.filters['jsonify'] = json.dumps
    return env
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#----------------------------------------------------------------------------#
# Imports.
#----------------------------------------------------------------------------#

# from flask import *  # do not use '*'; actually input the dependencies.
from flask import Flask, flash, redirect, Markup, g, render_template, request, url_for, Response
from flask.ext.sqlalchemy import SQLAlchemy
from flask.ext.bcrypt import Bcrypt
from flask_reggie import Reggie  # Regex Routing
import logging
from logging import Formatter, FileHandler
from forms import *
import os

import datetime
from flask.ext.login import LoginManager
from flask.ext.login import login_user, logout_user, login_required, current_user, fresh_login_required

# app specific
import models
# import config
# from carlae import config

#----------------------------------------------------------------------------#
# App Config.
#----------------------------------------------------------------------------#

app = Flask(__name__)
app.config.from_object('carlae.config')
db = SQLAlchemy(app)
bcrypt = Bcrypt(app)
Reggie(app)

login_manager = LoginManager()
login_manager.init_app(app)

# login manager settings.
login_manager.refresh_view = "login"
login_manager.needs_refresh_message = (
    u"To protect your account, please reauthenticate to access this page.")
login_manager.needs_refresh_message_category = "alert-info"

# user_loader callback
@login_manager.user_loader
def load_user(userid):
    return models.User.query.get(userid)

# @app.teardown_request
# def shutdown_session(exception=None):
#     db_session.remove()

#----------------------------------------------------------------------------#
# Controllers.
#----------------------------------------------------------------------------#

@app.before_request
def before_request():
    g.user = current_user

@app.route('/+', methods=['GET', 'POST'])
def home():
    # Display the URL shortcode form if there is authenticated user
    if g.user.is_authenticated():
        url_form = UrlForm(request.form)
        if url_form.validate_on_submit():
            url = models.Url(request.form['url'])
            url_key = '+' + url.shortlink
            short_url = os.path.join(config.BASE_URL, url_key)
            # build short_url as link & stylize here
            url_out = "
Here's your shortlink:
" \
                      "
%s
" % (short_url, short_url)
            flash(Markup(url_out), 'alert-success')
            return redirect(url_for('home'))
        return render_template("pages/index.html", form=url_form)
    else:
        login_form = LoginForm(request.form)
        if login_form.validate_on_submit():
            # login and validate the user...
            # set the boolean checkbox
            # http://nesv.blogspot.com/2011/10/flask-gotcha-with-html-forms-checkboxes.html
            remember_me = False
            if 'remember' in request.form:
                remember_me = True
            user = models.User(request.form['email'])
            login_user(user, remember=remember_me)
            return redirect(request.args.get("next") or url_for("home"))
        return render_template("pages/index.html", form=login_form)

# in case we need a fresh login
@app.route('/login', methods=['GET', 'POST'])
def login():
    login_form = LoginForm(request.form)
    if login_form.validate_on_submit():
        # login and validate the user...
        # set the boolean checkbox
        # http://nesv.blogspot.com/2011/10/flask-gotcha-with-html-forms-checkboxes.html
        remember_me = False
        if 'remember' in request.form:
            remember_me = True
        user = models.User(request.form['email'])
        login_user(user, remember=remember_me)
        return redirect(request.args.get("next") or url_for("home"))
    return render_template('forms/login.html', form=login_form)

@app.route("/logout")
@login_required
def logout():
    logout_user()  # logouts out user, deletes cookies, etc.
    return redirect(url_for('home'))

@app.route('/about')
def about():
    return render_template('pages/about.html')

@app.route('/invite', methods=['GET', 'POST'])
def invite():
    form = InviteUserForm(request.form)
    if request.method == 'POST':
        user = models.InviteUser(request.form['email'])
        if user.is_activated:
            message = Markup("%s has already activated an account" % user.email)
            flash(message, category='alert-danger')
            return redirect(url_for('invite'))
        try:
            mailgun = user.send_activation_email()
        except:
            message = Markup("There was an error sending an invitation to %s. Go ahead and try sending another invite."
                             % user.email)
            flash(message, category='alert-danger')
            return redirect(url_for('invite'))
        if mailgun.status_code != 200:
            message = Markup("There was an error sending an invitation to %s. Go ahead and try sending another invite."
                             % user.email)
            flash(message, category='alert-danger')
            return redirect(url_for('invite'))

        message = Markup("An invitation email has been sent to %s" % user.email)
        flash(message, category='alert-success')
        return redirect(url_for('invite'))

    return render_template('forms/invite.html', form = form)

@app.route('/activate', methods=['GET', 'POST'])
def activate():
    email = request.args.get('email')
    code = request.args.get('code')

    check_code = models.InviteUser.query.filter_by(activation_code=code).first()
    check_email = models.InviteUser.query.filter_by(email=email).first()
    check = check_email

    # Email & code must be in database
    if check_code is None or check_email is None:
        flash(Markup("Invalid activation code. Please request a new invitation."),
              category='alert-danger')
        return redirect(url_for('home'))
    else:
        check_code = check_code.activation_code
        check_email = check_email.email
    # Activation & email must match db values
    if not (check.activation_code==code) and (check_email==email):
        flash(Markup("Invalid activation code. Please request a new invitation."),
              category='alert-danger')
        return redirect(url_for('home'))

    # Activation must not be expired
    elapsed_time = datetime.datetime.now() - check.activation_date
    if elapsed_time > datetime.timedelta(weeks=2):
        # TODO: Set config with activation expiration option, currently 2 weeks.
        flash(Markup("Expired activation code. Please request a new invitation."),
              category='alert-danger')
        return redirect(url_for('home'))

    # Activate the form
    form = ActivateUserForm(request.form)
    if form.validate_on_submit():
        # now we create the new user
        usr = models.User(email)
        usr.create_user(request.form['password'])
        # change activation status
        inv = models.InviteUser.query.get(check.id)
        inv.is_activated = True
        models.db.session.commit()
        message = Markup("%s has been successfully activated. You're good to go!" % usr.email)
        flash(Markup(message), 'alert-success')
        return redirect(url_for('home'))
    return render_template('forms/activate.html', form=form, email=email)

### Shortlink Handler ###

@app.route('/<regex("[\+][0-9a-zA-Z]{6}"):shorturl>')
def example(shorturl):
    shorturl = shorturl.lstrip('+')
    url = models.ReverseUrl(shorturl)
    url.get_source_url()
    if url.source is not None:
        return redirect(url.source, code=301)  # permanent redirect
    return render_template('errors/404.html'), 404  # shortlink not found

# Raw DataBase Dump
@app.route('/rawdump')
def rawdump():
    urls =  models.Url.query.all()
    raw = ["shortlink, source_url"]
    for url in urls:
        raw.append("+%s, %s" %(url.shortlink, url.source))
    raw_string = ('\n').join(raw)
    return Response(raw_string, mimetype="text/plain")

@app.route('/top10')
def top10():
    top_urls = models.Url.query.order_by(models.Url.counter.desc()).limit(10).all()
    return render_template('pages/top10.html', urls=top_urls)

@app.route('/changepassword', methods=['GET', 'POST'])
@fresh_login_required
def changepassword():
    pass_form = ActivateUserForm(request.form)
    if pass_form.validate_on_submit():
        user = g.user
        user.change_password(request.form['password'])
        flash("Your password has been successfully changed!", 'alert-success')
        return redirect(url_for('home'))
    return render_template('forms/changepass.html', form=pass_form)

# Error handlers.

@app.errorhandler(500)
def internal_error(error):
    models.db.session.rollback()  # rollback db on 500 error
    return render_template('errors/500.html'), 500

@app.errorhandler(404)
def internal_error(error):
    return render_template('errors/404.html'), 404

if not app.debug:
    file_handler = FileHandler('error.log')
    file_handler.setFormatter(Formatter('%(asctime)s %(levelname)s: %(message)s '
    '[in %(pathname)s:%(lineno)d]'))
    app.logger.setLevel(logging.INFO)
    file_handler.setLevel(logging.INFO)
    app.logger.addHandler(file_handler)
    app.logger.info('errors')

#----------------------------------------------------------------------------#
# Launch.
#----------------------------------------------------------------------------#

# Default port:
if __name__ == '__main__':
    app.run(host='0.0.0.0')
<commit_msg>Added DEMO_MODE routing.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#----------------------------------------------------------------------------#
# Imports.
#----------------------------------------------------------------------------#

# from flask import *  # do not use '*'; actually input the dependencies.
from flask import Flask, flash, redirect, Markup, g, render_template, request, url_for, Response
from flask.ext.sqlalchemy import SQLAlchemy
from flask.ext.bcrypt import Bcrypt
from flask_reggie import Reggie  # Regex Routing
import logging
from logging import Formatter, FileHandler
from forms import *
import os

import datetime
from flask.ext.login import LoginManager
from flask.ext.login import login_user, logout_user, login_required, current_user, fresh_login_required

# app specific
import models
# import config
from carlae import config

#----------------------------------------------------------------------------#
# App Config.
#----------------------------------------------------------------------------#

app = Flask(__name__)
app.config.from_object('carlae.config')
db = SQLAlchemy(app)
bcrypt = Bcrypt(app)
Reggie(app)

login_manager = LoginManager()
login_manager.init_app(app)

# login manager settings.
login_manager.refresh_view = "login"
login_manager.needs_refresh_message = (
    u"To protect your account, please reauthenticate to access this page.")
login_manager.needs_refresh_message_category = "alert-info"

# user_loader callback
@login_manager.user_loader
def load_user(userid):
    return models.User.query.get(userid)

# @app.teardown_request
# def shutdown_session(exception=None):
#     db_session.remove()

#----------------------------------------------------------------------------#
# Controllers.
#----------------------------------------------------------------------------#

@app.before_request
def before_request():
    g.user = current_user

if config.DEMO_MODE:
    @app.route('/')
    def index():
        return redirect(url_for('home'))

@app.route('/+', methods=['GET', 'POST'])
def home():
    # Display the URL shortcode form if there is authenticated user
    if g.user.is_authenticated():
        url_form = UrlForm(request.form)
        if url_form.validate_on_submit():
            url = models.Url(request.form['url'])
            url_key = '+' + url.shortlink
            short_url = os.path.join(config.BASE_URL, url_key)
            # build short_url as link & stylize here
            url_out = "
Here's your shortlink:
" \
                      "
%s
" % (short_url, short_url)
            flash(Markup(url_out), 'alert-success')
            return redirect(url_for('home'))
        return render_template("pages/index.html", form=url_form)
    else:
        login_form = LoginForm(request.form)
        if login_form.validate_on_submit():
            # login and validate the user...
            # set the boolean checkbox
            # http://nesv.blogspot.com/2011/10/flask-gotcha-with-html-forms-checkboxes.html
            remember_me = False
            if 'remember' in request.form:
                remember_me = True
            user = models.User(request.form['email'])
            login_user(user, remember=remember_me)
            return redirect(request.args.get("next") or url_for("home"))
        return render_template("pages/index.html", form=login_form)

# in case we need a fresh login
@app.route('/login', methods=['GET', 'POST'])
def login():
    login_form = LoginForm(request.form)
    if login_form.validate_on_submit():
        # login and validate the user...
        # set the boolean checkbox
        # http://nesv.blogspot.com/2011/10/flask-gotcha-with-html-forms-checkboxes.html
        remember_me = False
        if 'remember' in request.form:
            remember_me = True
        user = models.User(request.form['email'])
        login_user(user, remember=remember_me)
        return redirect(request.args.get("next") or url_for("home"))
    return render_template('forms/login.html', form=login_form)

@app.route("/logout")
@login_required
def logout():
    logout_user()  # logouts out user, deletes cookies, etc.
    return redirect(url_for('home'))

@app.route('/about')
def about():
    return render_template('pages/about.html')

@app.route('/invite', methods=['GET', 'POST'])
def invite():
    form = InviteUserForm(request.form)
    if request.method == 'POST':
        user = models.InviteUser(request.form['email'])
        if user.is_activated:
            message = Markup("%s has already activated an account" % user.email)
            flash(message, category='alert-danger')
            return redirect(url_for('invite'))
        try:
            mailgun = user.send_activation_email()
        except:
            message = Markup("There was an error sending an invitation to %s. Go ahead and try sending another invite."
                             % user.email)
            flash(message, category='alert-danger')
            return redirect(url_for('invite'))
        if mailgun.status_code != 200:
            message = Markup("There was an error sending an invitation to %s. Go ahead and try sending another invite."
                             % user.email)
            flash(message, category='alert-danger')
            return redirect(url_for('invite'))

        message = Markup("An invitation email has been sent to %s" % user.email)
        flash(message, category='alert-success')
        return redirect(url_for('invite'))

    return render_template('forms/invite.html', form = form)

@app.route('/activate', methods=['GET', 'POST'])
def activate():
    email = request.args.get('email')
    code = request.args.get('code')

    check_code = models.InviteUser.query.filter_by(activation_code=code).first()
    check_email = models.InviteUser.query.filter_by(email=email).first()
    check = check_email

    # Email & code must be in database
    if check_code is None or check_email is None:
        flash(Markup("Invalid activation code. Please request a new invitation."),
              category='alert-danger')
        return redirect(url_for('home'))
    else:
        check_code = check_code.activation_code
        check_email = check_email.email
    # Activation & email must match db values
    if not (check.activation_code==code) and (check_email==email):
        flash(Markup("Invalid activation code. Please request a new invitation."),
              category='alert-danger')
        return redirect(url_for('home'))

    # Activation must not be expired
    elapsed_time = datetime.datetime.now() - check.activation_date
    if elapsed_time > datetime.timedelta(weeks=2):
        # TODO: Set config with activation expiration option, currently 2 weeks.
        flash(Markup("Expired activation code. Please request a new invitation."),
              category='alert-danger')
        return redirect(url_for('home'))

    # Activate the form
    form = ActivateUserForm(request.form)
    if form.validate_on_submit():
        # now we create the new user
        usr = models.User(email)
        usr.create_user(request.form['password'])
        # change activation status
        inv = models.InviteUser.query.get(check.id)
        inv.is_activated = True
        models.db.session.commit()
        message = Markup("%s has been successfully activated. You're good to go!" % usr.email)
        flash(Markup(message), 'alert-success')
        return redirect(url_for('home'))
    return render_template('forms/activate.html', form=form, email=email)

### Shortlink Handler ###

@app.route('/<regex("[\+][0-9a-zA-Z]{6}"):shorturl>')
def example(shorturl):
    shorturl = shorturl.lstrip('+')
    url = models.ReverseUrl(shorturl)
    url.get_source_url()
    if url.source is not None:
        return redirect(url.source, code=301)  # permanent redirect
    return render_template('errors/404.html'), 404  # shortlink not found

# Raw DataBase Dump
@app.route('/rawdump')
def rawdump():
    urls =  models.Url.query.all()
    raw = ["shortlink, source_url"]
    for url in urls:
        raw.append("+%s, %s" %(url.shortlink, url.source))
    raw_string = ('\n').join(raw)
    return Response(raw_string, mimetype="text/plain")

@app.route('/top10')
def top10():
    top_urls = models.Url.query.order_by(models.Url.counter.desc()).limit(10).all()
    return render_template('pages/top10.html', urls=top_urls)

@app.route('/changepassword', methods=['GET', 'POST'])
@fresh_login_required
def changepassword():
    pass_form = ActivateUserForm(request.form)
    if pass_form.validate_on_submit():
        user = g.user
        user.change_password(request.form['password'])
        flash("Your password has been successfully changed!", 'alert-success')
        return redirect(url_for('home'))
    return render_template('forms/changepass.html', form=pass_form)

# Error handlers.

@app.errorhandler(500)
def internal_error(error):
    models.db.session.rollback()  # rollback db on 500 error
    return render_template('errors/500.html'), 500

@app.errorhandler(404)
def internal_error(error):
    return render_template('errors/404.html'), 404

if not app.debug:
    file_handler = FileHandler('error.log')
    file_handler.setFormatter(Formatter('%(asctime)s %(levelname)s: %(message)s '
    '[in %(pathname)s:%(lineno)d]'))
    app.logger.setLevel(logging.INFO)
    file_handler.setLevel(logging.INFO)
    app.logger.addHandler(file_handler)
    app.logger.info('errors')

#----------------------------------------------------------------------------#
# Launch.
#----------------------------------------------------------------------------#

# Default port:
if __name__ == '__main__':
    app.run(host='0.0.0.0')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import ConfigParser
import wx
import time
from git_status import check_status

config = ConfigParser.ConfigParser()
config.readfp(open('config.cfg'))

REPOSITORY_PATH = config.get('main', 'repository_path')
REPEAT_EVERY = config.getfloat('main', 'repeat_every')

class Icon(wx.TaskBarIcon):
    """notifier's taskbar icon"""

    def __init__(self, menu):

        wx.TaskBarIcon.__init__(self)

        # menu options
        self.menu = menu

        # event handlers
        self.Bind(wx.EVT_TASKBAR_LEFT_DOWN, self.click)
        self.Bind(wx.EVT_TASKBAR_RIGHT_DOWN, self.click)
        self.Bind(wx.EVT_MENU, self.select)

        # icon state
        self.states = {
            "on": wx.Icon("dat/reader_new.png", wx.BITMAP_TYPE_PNG),
            "off": wx.Icon("dat/reader_empty.png", wx.BITMAP_TYPE_PNG)
        }
        self.setStatus("off")

    def click(self, event):
        """shows the menu"""

        menu = wx.Menu()
        for id, item in enumerate(self.menu):
            menu.Append(id, item[0])
        self.PopupMenu(menu)

    def select(self, event):
        """handles menu item selection"""

        self.menu[event.GetId()][1]()

    def setStatus(self, which):
        """sets the icon status"""

        self.SetIcon(self.states[which])

    def close(self):
        """destroys the icon"""

        self.Destroy()

class Popup(wx.Frame):
    """notifier's popup window"""

    def __init__(self):

        wx.Frame.__init__(self, None, -1, style=wx.NO_BORDER | wx.FRAME_NO_TASKBAR)
        self.padding = 12  # padding between edge, icon and text
        self.popped = 3  # the time popup was opened
        self.delay = 10  # time to leave the popup opened
        self.width = 400

        # platform specific hacks
        self.lineHeight = wx.MemoryDC().GetTextExtent(" ")[1]
        if wx.Platform == "__WXGTK__":
            # use the popup window widget on gtk as the
            # frame widget can't animate outside the screen
            self.popup = wx.PopupWindow(self, -1)
        elif wx.Platform == "__WXMSW__":
            # decrement line height on windows as the text calc below is off otherwise
            self.popup = self
            self.lineHeight -= 3
        elif wx.Platform == "__WXMAC__":
            # untested
            self.popup = self

        # main timer routine
        self.logo = wx.Bitmap("dat/reader_large.png")
        self.timer = wx.Timer(self, -1)
        self.Bind(wx.EVT_TIMER, self.main, self.timer)
        self.timer.Start(500)

    def adopt_size(self, text):
        lines = len(text.split("\n"))
        self.popup.SetSize((self.width, (self.lineHeight * (lines + 1)) + (self.padding * 2)))
        self.panel = wx.Panel(self.popup, -1, size=self.popup.GetSize())
        # popup's click handler
        self.panel.Bind(wx.EVT_LEFT_DOWN, self.click)

        # popup's logo
        wx.StaticBitmap(self.panel, -1, pos=(self.padding, self.padding)).SetBitmap(self.logo)

    def main(self, event):

        if self.focused():
            # maintain opened state if focused
            self.popped = time.time()
        elif self.opened() and self.popped + self.delay < time.time():
            # hide the popup once delay is reached
            self.hide()

    def click(self, event):
        """handles popup click"""

        self.popped = 0
        self.hide()

    def show(self, text):
        """shows the popup"""

        self.adopt_size(text)
        # create new text
        if hasattr(self, "text"):
            self.text.Destroy()
        popupSize = self.popup.GetSize()
        logoSize = self.logo.GetSize()
        self.text = wx.StaticText(self.panel, -1)
        self.text.SetLabel(text)
        self.text.Bind(wx.EVT_LEFT_DOWN, self.click)
        self.text.Move((logoSize.width + (self.padding * 2), self.padding))
        self.text.SetSize((
            popupSize.width - logoSize.width - (self.padding * 3),
            popupSize.height - (self.padding * 2)
        ))

        # animate the popup
        screen = wx.GetClientDisplayRect()
        self.popup.Show()
        for i in range(1, popupSize.height + 1):
            self.popup.Move((screen.width - popupSize.width, screen.height - i))
            self.popup.Update()
            self.popup.Refresh()
        self.popped = time.time()

    def hide(self):
        """hides the popup"""

        self.popup.Hide()
        self.popped = 0

    def focused(self):
        """returns true if popup has mouse focus"""

        mouse = wx.GetMousePosition()
        popup = self.popup.GetScreenRect()
        return (
            self.popped and
            mouse.x in range(popup.x, popup.x + popup.width)
            and mouse.y in range(popup.y, popup.y + popup.height)
        )

    def opened(self):
        """returns true if popup is open"""

        return self.popped != 0

class Notifier(wx.App):
    """main notifier app"""

    def __init__(self):

        wx.App.__init__(self, redirect=0)

        # menu handlers
        menu = [
            ("Show again", self.open_popup),
            ("Exit", self.exit),
        ]

        # main objects
        self.icon = Icon(menu)
        self.popup = Popup()
        # main timer routine
        timer = wx.Timer(self, -1)
        self.open_popup()
        self.Bind(wx.EVT_TIMER, self.main, timer)
        timer.Start(REPEAT_EVERY * 60000)
        self.MainLoop()

    def main(self, event):
        self.open_popup()

    def open_popup(self):
        wrap_text = lambda text, wrap_char: "%s\n%s\n\n" % (text, wrap_char * len(text))
        status = "off"
        message_str = ''
        for git_repo_path in REPOSITORY_PATH.split(','):
            result = check_status(git_repo_path)
            if result:
                message_str += wrap_text(git_repo_path, '=')
                for module, message in result.items():
                    message_str += "%s\n" % module
                    for k, v in message.items():
                        message_str += "\t%s\t%s\n\n" % (k, v)
                    status = "on"
        if message_str:
            self.popup.show(message_str)
        self.icon.setStatus(status)

    def exit(self):
        # close objects and end
        self.icon.close()
        self.Exit()

notifier = Notifier()
<commit_msg>Added run execution method<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import ConfigParser
import wx
import time
from git_status import check_status

config = ConfigParser.ConfigParser()
config.readfp(open('config.cfg'))

REPOSITORY_PATH = config.get('main', 'repository_path')
REPEAT_EVERY = config.getfloat('main', 'repeat_every')

class Icon(wx.TaskBarIcon):
    """notifier's taskbar icon"""

    def __init__(self, menu):

        wx.TaskBarIcon.__init__(self)

        # menu options
        self.menu = menu

        # event handlers
        self.Bind(wx.EVT_TASKBAR_LEFT_DOWN, self.click)
        self.Bind(wx.EVT_TASKBAR_RIGHT_DOWN, self.click)
        self.Bind(wx.EVT_MENU, self.select)

        # icon state
        self.states = {
            "on": wx.Icon("dat/reader_new.png", wx.BITMAP_TYPE_PNG),
            "off": wx.Icon("dat/reader_empty.png", wx.BITMAP_TYPE_PNG)
        }
        self.setStatus("off")

    def click(self, event):
        """shows the menu"""

        menu = wx.Menu()
        for id, item in enumerate(self.menu):
            menu.Append(id, item[0])
        self.PopupMenu(menu)

    def select(self, event):
        """handles menu item selection"""

        self.menu[event.GetId()][1]()

    def setStatus(self, which):
        """sets the icon status"""

        self.SetIcon(self.states[which])

    def run_method(method):
        return locals()['%s' % method]()

    def close(self):
        """destroys the icon"""

        self.Destroy()

class Popup(wx.Frame):
    """notifier's popup window"""

    def __init__(self):

        wx.Frame.__init__(self, None, -1, style=wx.NO_BORDER | wx.FRAME_NO_TASKBAR)
        self.padding = 12  # padding between edge, icon and text
        self.popped = 3  # the time popup was opened
        self.delay = 10  # time to leave the popup opened
        self.width = 400

        # platform specific hacks
        self.lineHeight = wx.MemoryDC().GetTextExtent(" ")[1]
        if wx.Platform == "__WXGTK__":
            # use the popup window widget on gtk as the
            # frame widget can't animate outside the screen
            self.popup = wx.PopupWindow(self, -1)
        elif wx.Platform == "__WXMSW__":
            # decrement line height on windows as the text calc below is off otherwise
            self.popup = self
            self.lineHeight -= 3
        elif wx.Platform == "__WXMAC__":
            # untested
            self.popup = self

        # main timer routine
        self.logo = wx.Bitmap("dat/reader_large.png")
        self.timer = wx.Timer(self, -1)
        self.Bind(wx.EVT_TIMER, self.main, self.timer)
        self.timer.Start(500)

    def adopt_size(self, text):
        lines = len(text.split("\n"))
        self.popup.SetSize((self.width, (self.lineHeight * (lines + 1)) + (self.padding * 2)))
        self.panel = wx.Panel(self.popup, -1, size=self.popup.GetSize())
        # popup's click handler
        self.panel.Bind(wx.EVT_LEFT_DOWN, self.click)

        # popup's logo
        wx.StaticBitmap(self.panel, -1, pos=(self.padding, self.padding)).SetBitmap(self.logo)

    def main(self, event):

        if self.focused():
            # maintain opened state if focused
            self.popped = time.time()
        elif self.opened() and self.popped + self.delay < time.time():
            # hide the popup once delay is reached
            self.hide()

    def click(self, event):
        """handles popup click"""

        self.popped = 0
        self.hide()

    def show(self, text):
        """shows the popup"""

        self.adopt_size(text)
        # create new text
        if hasattr(self, "text"):
            self.text.Destroy()
        popupSize = self.popup.GetSize()
        logoSize = self.logo.GetSize()
        self.text = wx.StaticText(self.panel, -1)
        self.text.SetLabel(text)
        self.text.Bind(wx.EVT_LEFT_DOWN, self.click)
        self.text.Move((logoSize.width + (self.padding * 2), self.padding))
        self.text.SetSize((
            popupSize.width - logoSize.width - (self.padding * 3),
            popupSize.height - (self.padding * 2)
        ))

        # animate the popup
        screen = wx.GetClientDisplayRect()
        self.popup.Show()
        for i in range(1, popupSize.height + 1):
            self.popup.Move((screen.width - popupSize.width, screen.height - i))
            self.popup.Update()
            self.popup.Refresh()
        self.popped = time.time()

    def hide(self):
        """hides the popup"""

        self.popup.Hide()
        self.popped = 0

    def focused(self):
        """returns true if popup has mouse focus"""

        mouse = wx.GetMousePosition()
        popup = self.popup.GetScreenRect()
        return (
            self.popped and
            mouse.x in range(popup.x, popup.x + popup.width)
            and mouse.y in range(popup.y, popup.y + popup.height)
        )

    def opened(self):
        """returns true if popup is open"""

        return self.popped != 0

class Notifier(wx.App):
    """main notifier app"""

    def __init__(self):

        wx.App.__init__(self, redirect=0)

        # menu handlers
        menu = [
            ("Show again", self.open_popup),
            ("Exit", self.exit),
        ]

        # main objects
        self.icon = Icon(menu)
        self.popup = Popup()
        # main timer routine
        timer = wx.Timer(self, -1)
        self.open_popup()
        self.Bind(wx.EVT_TIMER, self.main, timer)
        timer.Start(REPEAT_EVERY * 60000)
        self.MainLoop()

    def main(self, event):
        self.open_popup()

    def open_popup(self):
        wrap_text = lambda text, wrap_char: "%s\n%s\n\n" % (text, wrap_char * len(text))
        status = "off"
        message_str = ''
        for git_repo_path in REPOSITORY_PATH.split(','):
            result = check_status(git_repo_path)
            if result:
                message_str += wrap_text(git_repo_path, '=')
                for module, message in result.items():
                    message_str += "%s\n" % module
                    for k, v in message.items():
                        message_str += "\t%s\t%s\n\n" % (k, v)
                    status = "on"
        if message_str:
            self.popup.show(message_str)
        self.icon.setStatus(status)

    def exit(self):
        # close objects and end
        self.icon.close()
        self.Exit()

notifier = Notifier()
<|endoftext|>


<commit_before>import tempfile
import bloscpack
import blosc
import pickle
import os
from os import mkdir
from os.path import exists, isdir, join
import pandas as pd
import numpy as np
from pandas import msgpack
import shutil
from pandas.core.indexing import convert_to_index_sliceable

def escape(text):
    return str(text)

def _safe_mkdir(path):
    if not exists(path):
        mkdir(path)

class Castra(object):
    def __init__(self, path=None, template=None):
        # check if we should create a random path
        if path is None:
            self.path = tempfile.mkdtemp(prefix='castra-')
            self._explicitly_given_path = False
        else:
            self.path = path
            self._explicitly_given_path = True

        # check if the given path exists already and create it if it doesn't
        if not exists(self.path):
            mkdir(self.path)
        # raise an Exception if it isn't a directory
        elif not isdir(self.path):
            raise ValueError("'path': %s must be a directory")

        self.meta_path = self.dirname('meta')

        # either we have a meta directory
        if exists(self.meta_path) and isdir(self.meta_path):
            if template is not None:
                raise ValueError(
                    "'template' must be 'None' when opening a Castra")
            self.load_meta()
            self.load_partitions()
        # or we don't, in which case we need a template
        elif template is not None:
            mkdir(self.meta_path)
            self.columns, self.dtypes, self.index_dtype = \
                list(template.columns), template.dtypes, template.index.dtype
            self.partitions = pd.Series([], dtype='O',
                                        index=template.index.__class__([]))
            self.flush_meta()
            self.save_partitions()
        else:
            raise ValueError(
                "must specify a 'template' when creating a new Castra")

    def load_meta(self, loads=pickle.loads):
        meta = []
        for name in ['columns', 'dtypes', 'index_dtype']:
            with open(join(self.meta_path, name), 'r') as f:
                meta.append(loads(f.read()))
        self.columns, self.dtype, self.index_dtype = meta

    def flush_meta(self, dumps=pickle.dumps):
        for name in ['columns', 'dtypes', 'index_dtype']:
            with open(join(self.meta_path, name), 'w') as f:
                f.write(dumps(getattr(self, name)))

    def load_partitions(self, loads=pickle.loads):
        with open(join(self.meta_path, 'plist'), 'r') as f:
            self.partitions = pickle.loads(f.read())

    def save_partitions(self, dumps=pickle.dumps):
        with open(join(self.meta_path, 'plist'), 'w') as f:
            f.write(dumps(self.partitions))

    def extend(self, df):
        # TODO: Ensure that df is consistent with existing data
        index = df.index.values
        partition_name = '--'.join([escape(index.min()), escape(index.max())])

        mkdir(self.dirname(partition_name))

        # Store columns
        for col in df.columns:
            fn = self.dirname(partition_name, col)
            x = df[col].values
            pack_file(x, fn)

        # Store index
        fn = self.dirname(partition_name, '.index')
        x = df.index.values
        bloscpack.pack_ndarray_file(x, fn)

        self.partitions[index.max()] = partition_name

    def dirname(self, *args):
        return os.path.join(self.path, *args)

    def load_partition(self, name):
        columns = [unpack_file(self.dirname(name, col))
                   for col in self.columns]
        index = unpack_file(self.dirname(name, '.index'))

        return pd.DataFrame(dict(zip(self.columns, columns)),
                            columns=self.columns,
                            index=pd.Index(index, dtype=self.index_dtype))

    def __getitem__(self, key):
        assert isinstance(key, slice)
        start, stop = key.start, key.stop
        names = select_partitions(self.partitions, key)

        data_frames = [self.load_partition(name) for name in names]

        data_frames[0] = data_frames[0].loc[start:]
        data_frames[-1] = data_frames[-1].loc[:stop]
        return pd.concat(data_frames)

    def drop(self):
        if os.path.exists(self.path):
            shutil.rmtree(self.path)

    def flush(self):
        self.save_partitions()

    def __enter__(self):
        return self

    def __exit__(self, *args):
        if not self._explicitly_given_path:
            self.drop()
        else:
            self.save_partitions()

    def __del__(self):
        if not self._explicitly_given_path:
            self.drop()

    def __getstate__(self):
        self.save_partitions()
        return (self.path, self._explicitly_given_path)

    def __setstate__(self, state):
        self.path = state[0]
        self._explicitly_given_path = state[1]
        self.meta_path = self.dirname('meta')
        self.load_meta()
        self.load_partitions()

def pack_file(x, fn):
    """ Pack numpy array into filename

    Supports binary data with bloscpack and text data with msgpack+blosc

    >>> pack_file(np.array([1, 2, 3]), 'foo.blp')  # doctest: +SKIP

    See also:
        unpack_file
    """
    if x.dtype != 'O':
        bloscpack.pack_ndarray_file(x, fn)
    else:
        bytes = blosc.compress(msgpack.packb(x.tolist()), 1)
        with open(fn, 'wb') as f:
            f.write(bytes)

def unpack_file(fn):
    """ Unpack numpy array from filename

    Supports binary data with bloscpack and text data with msgpack+blosc

    >>> unpack_file('foo.blp')  # doctest: +SKIP
    array([1, 2, 3])

    See also:
        pack_file
    """
    try:
        return bloscpack.unpack_ndarray_file(fn)
    except ValueError:
        with open(fn, 'rb') as f:
            bytes = f.read()
        return np.array(msgpack.unpackb(blosc.decompress(bytes)))

def coerce_index(dt, o):
    if np.issubdtype(dt, np.datetime64):
        return pd.Timestamp(o)
    return o

def select_partitions(partitions, key):
    """ Select partitions from partition list given slice

    >>> p = pd.Series(['a', 'b', 'c', 'd', 'e'], index=[0, 10, 20, 30, 40])
    >>> select_partitions(p, slice(3, 25))
    ['b', 'c', 'd']
    """
    assert key.step is None
    start, stop = key.start, key.stop
    names = list(partitions.loc[start:stop])

    last = partitions.searchsorted(names[-1])[0]

    stop2 = coerce_index(partitions.index.dtype, stop)
    if partitions.index[last] < stop2 and len(partitions) > last + 1:
        names.append(partitions.iloc[last + 1])

    return names
<commit_msg>remove unused import<commit_after>import tempfile
import bloscpack
import blosc
import pickle
import os
from os import mkdir
from os.path import exists, isdir, join
import pandas as pd
import numpy as np
from pandas import msgpack
import shutil

def escape(text):
    return str(text)

def _safe_mkdir(path):
    if not exists(path):
        mkdir(path)

class Castra(object):
    def __init__(self, path=None, template=None):
        # check if we should create a random path
        if path is None:
            self.path = tempfile.mkdtemp(prefix='castra-')
            self._explicitly_given_path = False
        else:
            self.path = path
            self._explicitly_given_path = True

        # check if the given path exists already and create it if it doesn't
        if not exists(self.path):
            mkdir(self.path)
        # raise an Exception if it isn't a directory
        elif not isdir(self.path):
            raise ValueError("'path': %s must be a directory")

        self.meta_path = self.dirname('meta')

        # either we have a meta directory
        if exists(self.meta_path) and isdir(self.meta_path):
            if template is not None:
                raise ValueError(
                    "'template' must be 'None' when opening a Castra")
            self.load_meta()
            self.load_partitions()
        # or we don't, in which case we need a template
        elif template is not None:
            mkdir(self.meta_path)
            self.columns, self.dtypes, self.index_dtype = \
                list(template.columns), template.dtypes, template.index.dtype
            self.partitions = pd.Series([], dtype='O',
                                        index=template.index.__class__([]))
            self.flush_meta()
            self.save_partitions()
        else:
            raise ValueError(
                "must specify a 'template' when creating a new Castra")

    def load_meta(self, loads=pickle.loads):
        meta = []
        for name in ['columns', 'dtypes', 'index_dtype']:
            with open(join(self.meta_path, name), 'r') as f:
                meta.append(loads(f.read()))
        self.columns, self.dtype, self.index_dtype = meta

    def flush_meta(self, dumps=pickle.dumps):
        for name in ['columns', 'dtypes', 'index_dtype']:
            with open(join(self.meta_path, name), 'w') as f:
                f.write(dumps(getattr(self, name)))

    def load_partitions(self, loads=pickle.loads):
        with open(join(self.meta_path, 'plist'), 'r') as f:
            self.partitions = pickle.loads(f.read())

    def save_partitions(self, dumps=pickle.dumps):
        with open(join(self.meta_path, 'plist'), 'w') as f:
            f.write(dumps(self.partitions))

    def extend(self, df):
        # TODO: Ensure that df is consistent with existing data
        index = df.index.values
        partition_name = '--'.join([escape(index.min()), escape(index.max())])

        mkdir(self.dirname(partition_name))

        # Store columns
        for col in df.columns:
            fn = self.dirname(partition_name, col)
            x = df[col].values
            pack_file(x, fn)

        # Store index
        fn = self.dirname(partition_name, '.index')
        x = df.index.values
        bloscpack.pack_ndarray_file(x, fn)

        self.partitions[index.max()] = partition_name

    def dirname(self, *args):
        return os.path.join(self.path, *args)

    def load_partition(self, name):
        columns = [unpack_file(self.dirname(name, col))
                   for col in self.columns]
        index = unpack_file(self.dirname(name, '.index'))

        return pd.DataFrame(dict(zip(self.columns, columns)),
                            columns=self.columns,
                            index=pd.Index(index, dtype=self.index_dtype))

    def __getitem__(self, key):
        assert isinstance(key, slice)
        start, stop = key.start, key.stop
        names = select_partitions(self.partitions, key)

        data_frames = [self.load_partition(name) for name in names]

        data_frames[0] = data_frames[0].loc[start:]
        data_frames[-1] = data_frames[-1].loc[:stop]
        return pd.concat(data_frames)

    def drop(self):
        if os.path.exists(self.path):
            shutil.rmtree(self.path)

    def flush(self):
        self.save_partitions()

    def __enter__(self):
        return self

    def __exit__(self, *args):
        if not self._explicitly_given_path:
            self.drop()
        else:
            self.save_partitions()

    def __del__(self):
        if not self._explicitly_given_path:
            self.drop()

    def __getstate__(self):
        self.save_partitions()
        return (self.path, self._explicitly_given_path)

    def __setstate__(self, state):
        self.path = state[0]
        self._explicitly_given_path = state[1]
        self.meta_path = self.dirname('meta')
        self.load_meta()
        self.load_partitions()

def pack_file(x, fn):
    """ Pack numpy array into filename

    Supports binary data with bloscpack and text data with msgpack+blosc

    >>> pack_file(np.array([1, 2, 3]), 'foo.blp')  # doctest: +SKIP

    See also:
        unpack_file
    """
    if x.dtype != 'O':
        bloscpack.pack_ndarray_file(x, fn)
    else:
        bytes = blosc.compress(msgpack.packb(x.tolist()), 1)
        with open(fn, 'wb') as f:
            f.write(bytes)

def unpack_file(fn):
    """ Unpack numpy array from filename

    Supports binary data with bloscpack and text data with msgpack+blosc

    >>> unpack_file('foo.blp')  # doctest: +SKIP
    array([1, 2, 3])

    See also:
        pack_file
    """
    try:
        return bloscpack.unpack_ndarray_file(fn)
    except ValueError:
        with open(fn, 'rb') as f:
            bytes = f.read()
        return np.array(msgpack.unpackb(blosc.decompress(bytes)))

def coerce_index(dt, o):
    if np.issubdtype(dt, np.datetime64):
        return pd.Timestamp(o)
    return o

def select_partitions(partitions, key):
    """ Select partitions from partition list given slice

    >>> p = pd.Series(['a', 'b', 'c', 'd', 'e'], index=[0, 10, 20, 30, 40])
    >>> select_partitions(p, slice(3, 25))
    ['b', 'c', 'd']
    """
    assert key.step is None
    start, stop = key.start, key.stop
    names = list(partitions.loc[start:stop])

    last = partitions.searchsorted(names[-1])[0]

    stop2 = coerce_index(partitions.index.dtype, stop)
    if partitions.index[last] < stop2 and len(partitions) > last + 1:
        names.append(partitions.iloc[last + 1])

    return names
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from openerp import http, _
from openerp.http import request
from openerp.addons.base.ir.ir_qweb import AssetsBundle
from openerp.addons.web.controllers.main import binary_content
import base64

class LivechatController(http.Controller):

    @http.route('/im_livechat/external_lib.<any(css,js):ext>', type='http', auth='none')
    def livechat_lib(self, ext, **kwargs):
        asset = AssetsBundle("im_livechat.external_lib")
        mock_attachment = getattr(asset, ext)()
        if isinstance(mock_attachment, list):  # suppose that CSS asset will not required to be split in pages
            mock_attachment = mock_attachment[0]
        attachment = request.env['ir.attachment'].search([('name', '=', mock_attachment.url)])
        # can't use /web/content directly because we don't have attachment ids (attachments must be created)
        status, headers, content = binary_content(id=attachment.id, unique=asset.checksum)
        content_base64 = base64.b64decode(content)
        headers.append(('Content-Length', len(content_base64)))
        return request.make_response(content_base64, headers)

    @http.route('/im_livechat/support/<int:channel_id>', type='http', auth='public')
    def support_page(self, channel_id, **kwargs):
        channel = request.env['im_livechat.channel'].sudo().browse(channel_id)
        return request.render('im_livechat.support_page', {'channel': channel})

    @http.route('/im_livechat/loader/<int:channel_id>', type='http', auth='public')
    def loader(self, channel_id, **kwargs):
        username = kwargs.get("username", _("Visitor"))
        channel = request.env['im_livechat.channel'].sudo().browse(channel_id)
        info = request.env['im_livechat.channel'].match_rules(request, channel.id, username=username)
        return request.render('im_livechat.loader', {'info': info, 'web_session_required': True}) if info else False

    @http.route('/im_livechat/get_session', type="json", auth='public')
    def get_session(self, channel_id, anonymous_name, **kwargs):
        # if geoip, add the country name to the anonymous name
        if request.session.geoip:
            anonymous_name = anonymous_name + " ("+request.session.geoip.get('country_name', "")+")"
        # if the user is identifiy (eg: portal user on the frontend), don't use the anonymous name. The user will be added to session.
        if request.session.uid:
            anonymous_name = request.env.user.name
        return request.env["im_livechat.channel"].get_mail_channel(channel_id, anonymous_name)

    @http.route('/im_livechat/feedback', type='json', auth='public')
    def feedback(self, uuid, rate, reason=None, **kwargs):
        Channel = request.env['mail.channel']
        Rating = request.env['rating.rating']
        channel = Channel.sudo().search([('uuid', '=', uuid)], limit=1)
        if channel:
            # limit the creation : only ONE rating per session
            values = {
                'rating': rate,
            }
            if not channel.rating_ids:
                values.update({
                    'res_id': channel.id,
                    'res_model': 'mail.channel',
                    'rating': rate,
                    'feedback': reason,
                })
                # find the partner (operator)
                if channel.channel_partner_ids:
                    values['rated_partner_id'] = channel.channel_partner_ids[0] and channel.channel_partner_ids[0].id or False
                # create the rating
                rating = Rating.sudo().create(values)
            else:
                if reason:
                    values['feedback'] = reason
                rating = channel.rating_ids[0]
                rating.write(values)
            return rating.id
        return False
<commit_msg>[FIX] im_livechat : assets controller, mock_attachment includes id now.<commit_after># -*- coding: utf-8 -*-
from openerp import http, _
from openerp.http import request
from openerp.addons.base.ir.ir_qweb import AssetsBundle
from openerp.addons.web.controllers.main import binary_content
import base64

class LivechatController(http.Controller):

    @http.route('/im_livechat/external_lib.<any(css,js):ext>', type='http', auth='none')
    def livechat_lib(self, ext, **kwargs):
        asset = AssetsBundle("im_livechat.external_lib")
        mock_attachment = getattr(asset, ext)()
        if isinstance(mock_attachment, list):  # suppose that CSS asset will not required to be split in pages
            mock_attachment = mock_attachment[0]
        # can't use /web/content directly because we don't have attachment ids (attachments must be created)
        status, headers, content = binary_content(id=mock_attachment.id, unique=asset.checksum)
        content_base64 = base64.b64decode(content) if content else ''
        headers.append(('Content-Length', len(content_base64)))
        return request.make_response(content_base64, headers)

    @http.route('/im_livechat/support/<int:channel_id>', type='http', auth='public')
    def support_page(self, channel_id, **kwargs):
        channel = request.env['im_livechat.channel'].sudo().browse(channel_id)
        return request.render('im_livechat.support_page', {'channel': channel})

    @http.route('/im_livechat/loader/<int:channel_id>', type='http', auth='public')
    def loader(self, channel_id, **kwargs):
        username = kwargs.get("username", _("Visitor"))
        channel = request.env['im_livechat.channel'].sudo().browse(channel_id)
        info = request.env['im_livechat.channel'].match_rules(request, channel.id, username=username)
        return request.render('im_livechat.loader', {'info': info, 'web_session_required': True}) if info else False

    @http.route('/im_livechat/get_session', type="json", auth='public')
    def get_session(self, channel_id, anonymous_name, **kwargs):
        # if geoip, add the country name to the anonymous name
        if request.session.geoip:
            anonymous_name = anonymous_name + " ("+request.session.geoip.get('country_name', "")+")"
        # if the user is identifiy (eg: portal user on the frontend), don't use the anonymous name. The user will be added to session.
        if request.session.uid:
            anonymous_name = request.env.user.name
        return request.env["im_livechat.channel"].get_mail_channel(channel_id, anonymous_name)

    @http.route('/im_livechat/feedback', type='json', auth='public')
    def feedback(self, uuid, rate, reason=None, **kwargs):
        Channel = request.env['mail.channel']
        Rating = request.env['rating.rating']
        channel = Channel.sudo().search([('uuid', '=', uuid)], limit=1)
        if channel:
            # limit the creation : only ONE rating per session
            values = {
                'rating': rate,
            }
            if not channel.rating_ids:
                values.update({
                    'res_id': channel.id,
                    'res_model': 'mail.channel',
                    'rating': rate,
                    'feedback': reason,
                })
                # find the partner (operator)
                if channel.channel_partner_ids:
                    values['rated_partner_id'] = channel.channel_partner_ids[0] and channel.channel_partner_ids[0].id or False
                # create the rating
                rating = Rating.sudo().create(values)
            else:
                if reason:
                    values['feedback'] = reason
                rating = channel.rating_ids[0]
                rating.write(values)
            return rating.id
        return False
<|endoftext|>