<commit_before>import paramiko
import time
import re
from netmiko.cisco_base_connection import CiscoSSHConnection

class FortinetSSH(CiscoSSHConnection):
    def _modify_connection_params(self):
        """Modify connection parameters prior to SSH connection."""
        paramiko.Transport._preferred_kex = (
            "diffie-hellman-group14-sha1",
            "diffie-hellman-group-exchange-sha1",
            "diffie-hellman-group-exchange-sha256",
            "diffie-hellman-group1-sha1",
        )

    def session_preparation(self):
        """Prepare the session after the connection has been established."""
        delay_factor = self.select_delay_factor(delay_factor=0)
        output = ""

        # If "set post-login-banner enable" is set it will require you to press 'a'
        # to accept the banner before you login. This will accept if it occurs
        count = 1
        while count <= 30:
            output += self.read_channel()
            if "to accept" in output:
                self.write_channel("a\r")
                break
            else:
                time.sleep(0.33 * delay_factor)
            count += 1

        self._test_channel_read()
        self.set_base_prompt(alt_prompt_terminator="$")
        self.disable_paging()
        # Clear the read buffer
        time.sleep(0.3 * self.global_delay_factor)
        self.clear_buffer()

    def disable_paging(self, delay_factor=1, **kwargs):
        """Disable paging is only available with specific roles so it may fail."""
        check_command = "get system status | grep Virtual"
        output = self.send_command_timing(check_command)
        self.allow_disable_global = True
        self.vdoms = False
        self._output_mode = "more"

        if "Virtual domain configuration: enable" in output:
            self.vdoms = True
            vdom_additional_command = "config global"
            output = self.send_command_timing(vdom_additional_command, delay_factor=2)
            if "Command fail" in output:
                self.allow_disable_global = False
                self.remote_conn.close()
                self.establish_connection(width=100, height=1000)

        new_output = ""
        if self.allow_disable_global:
            self._retrieve_output_mode()
            disable_paging_commands = [
                "config system console",
                "set output standard",
                "end",
            ]
            # There is an extra 'end' required if in multi-vdoms are enabled
            if self.vdoms:
                disable_paging_commands.append("end")
            outputlist = [
                self.send_command_timing(command, delay_factor=2)
                for command in disable_paging_commands
            ]
            # Should test output is valid
            new_output = self.RETURN.join(outputlist)

        return output + new_output

    def _retrieve_output_mode(self):
        """Save the state of the output mode so it can be reset at the end of the session."""
        reg_mode = re.compile(r"output\s+:\s+(?P<mode>.*)\s+\n")
        output = self.send_command("get system console")
        result_mode_re = reg_mode.search(output)
        if result_mode_re:
            result_mode = result_mode_re.group("mode").strip()
            if result_mode in ["more", "standard"]:
                self._output_mode = result_mode

    def cleanup(self, command="exit"):
        """Re-enable paging globally."""
        if self.allow_disable_global:
            # Return paging state
            output_mode_cmd = f"set output {self._output_mode}"
            enable_paging_commands = ["config system console", output_mode_cmd, "end"]
            if self.vdoms:
                enable_paging_commands.insert(0, "config global")
            # Should test output is valid
            for command in enable_paging_commands:
                self.send_command_timing(command)
        return super().cleanup(command=command)

    def config_mode(self, config_command=""):
        """No config mode for Fortinet devices."""
        return ""

    def exit_config_mode(self, exit_config=""):
        """No config mode for Fortinet devices."""
        return ""

    def save_config(self, *args, **kwargs):
        """Not Implemented"""
        raise NotImplementedError
<commit_msg>Update fortinet_ssh.py (#1968)<commit_after>import paramiko
import time
import re
from netmiko.cisco_base_connection import CiscoSSHConnection

class FortinetSSH(CiscoSSHConnection):
    def _modify_connection_params(self):
        """Modify connection parameters prior to SSH connection."""
        paramiko.Transport._preferred_kex = (
            "diffie-hellman-group14-sha1",
            "diffie-hellman-group-exchange-sha1",
            "diffie-hellman-group-exchange-sha256",
            "diffie-hellman-group1-sha1",
        )

    def session_preparation(self):
        """Prepare the session after the connection has been established."""
        delay_factor = self.select_delay_factor(delay_factor=0)
        output = ""

        # If "set post-login-banner enable" is set it will require you to press 'a'
        # to accept the banner before you login. This will accept if it occurs
        count = 1
        while count <= 30:
            output += self.read_channel()
            if "to accept" in output:
                self.write_channel("a\r")
                break
            else:
                time.sleep(0.33 * delay_factor)
            count += 1

        self._test_channel_read()
        self.set_base_prompt(alt_prompt_terminator="$")
        self.disable_paging()
        # Clear the read buffer
        time.sleep(0.3 * self.global_delay_factor)
        self.clear_buffer()

    def disable_paging(self, delay_factor=1, **kwargs):
        """Disable paging is only available with specific roles so it may fail."""
        check_command = "get system status | grep Virtual"
        output = self.send_command_timing(check_command)
        self.allow_disable_global = True
        self.vdoms = False
        self._output_mode = "more"

        if re.search(r"Virtual domain configuration: (multiple|enable)", output):
            self.vdoms = True
            vdom_additional_command = "config global"
            output = self.send_command_timing(vdom_additional_command, delay_factor=2)
            if "Command fail" in output:
                self.allow_disable_global = False
                self.remote_conn.close()
                self.establish_connection(width=100, height=1000)

        new_output = ""
        if self.allow_disable_global:
            self._retrieve_output_mode()
            disable_paging_commands = [
                "config system console",
                "set output standard",
                "end",
            ]
            # There is an extra 'end' required if in multi-vdoms are enabled
            if self.vdoms:
                disable_paging_commands.append("end")
            outputlist = [
                self.send_command_timing(command, delay_factor=2)
                for command in disable_paging_commands
            ]
            # Should test output is valid
            new_output = self.RETURN.join(outputlist)

        return output + new_output

    def _retrieve_output_mode(self):
        """Save the state of the output mode so it can be reset at the end of the session."""
        reg_mode = re.compile(r"output\s+:\s+(?P<mode>.*)\s+\n")
        output = self.send_command("get system console")
        result_mode_re = reg_mode.search(output)
        if result_mode_re:
            result_mode = result_mode_re.group("mode").strip()
            if result_mode in ["more", "standard"]:
                self._output_mode = result_mode

    def cleanup(self, command="exit"):
        """Re-enable paging globally."""
        if self.allow_disable_global:
            # Return paging state
            output_mode_cmd = f"set output {self._output_mode}"
            enable_paging_commands = ["config system console", output_mode_cmd, "end"]
            if self.vdoms:
                enable_paging_commands.insert(0, "config global")
            # Should test output is valid
            for command in enable_paging_commands:
                self.send_command_timing(command)
        return super().cleanup(command=command)

    def config_mode(self, config_command=""):
        """No config mode for Fortinet devices."""
        return ""

    def exit_config_mode(self, exit_config=""):
        """No config mode for Fortinet devices."""
        return ""

    def save_config(self, *args, **kwargs):
        """Not Implemented"""
        raise NotImplementedError
<|endoftext|>


<commit_before>"""
batch/evol.py

Code for adaptive stochastic descent optimizaiton method (Kerr et al. 2018; PLoS ONE)

Contributors: salvadordura@gmail.com
"""

from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import

from builtins import zip

from builtins import range
from builtins import open
from builtins import str
from future import standard_library
standard_library.install_aliases()

# required to make json saving work in Python 2/3
try:
    to_unicode = unicode
except NameError:
    to_unicode = str

import imp
import json
import logging
import datetime
import os
import signal
import glob
from copy import copy
from random import Random
from time import sleep, time
from itertools import product
from subprocess import Popen, PIPE
import importlib, types
import numpy.random as nr
import numpy as np
import pickle

from neuron import h
from netpyne import sim, specs
import optuna
from .utils import createFolder
from .utils import bashTemplate
from .utils import dcp, sigfig

pc = h.ParallelContext() # use bulletin board master/slave

# -------------------------------------------------------------------------------
# Optuna optimization
# -------------------------------------------------------------------------------

# func needs to be outside of class
def runJob(script, cfgSavePath, netParamsSavePath, simDataPath):
    import os
    print('\nJob in rank id: ',pc.id())
    command = 'nrniv %s simConfig=%s netParams=%s' % (script, cfgSavePath, netParamsSavePath)
    print(command)

    with open(simDataPath+'.run', 'w') as outf, open(simDataPath+'.err', 'w') as errf:
        pid = Popen(command.split(' '), stdout=outf, stderr=errf, preexec_fn=os.setsid).pid

    with open('./pids.pid', 'a') as file:
        file.write(str(pid) + ' ')

def optunaOptim(self, pc):
    import sys

    # -------------------------------------------------------------------------------
    # Optuna optimization: Parallel evaluation
    # -------------------------------------------------------------------------------
    def objective(trial, args):

        import os

        ngen = trial.number
        total_jobs = 0

        # options slurm, mpi
        type = args.get('type', 'mpi_direct')

        # params
        paramLabels = args.get('paramLabels', [])
        minVals = args.get('minVals', [])
        maxVals = args.get('maxVals', [])

        # paths to required scripts
        script = args.get('script', 'init.py')
        netParamsSavePath =  args.get('netParamsSavePath')
        genFolderPath = self.saveFolder + '/trial_' + str(ngen)

        # mpi command setup
        nodes = args.get('nodes', 1)
        coresPerNode = args.get('coresPerNode', 1)
        mpiCommand = args.get('mpiCommand', 'ibrun')
        numproc = nodes*coresPerNode

        # slurm setup
        custom = args.get('custom', '')
        folder = args.get('folder', '.')
        email = args.get('email', 'a@b.c')
        walltime = args.get('walltime', '00:01:00')
        reservation = args.get('reservation', None)
        allocation = args.get('allocation', 'csd403') # NSG account

        # fitness function
        fitnessFunc = args.get('fitnessFunc')
        fitnessFuncArgs = args.get('fitnessFuncArgs')
        maxFitness = args.get('maxFitness')

        # read params or set defaults
        sleepInterval = args.get('sleepInterval', 0.2)

        # create folder if it does not exist
        createFolder(genFolderPath)

        # --------------------------------------
        # generate param values for optuna trial
        candidate = []
        for paramLabel, minVal, maxVal in zip(paramLabels, minVals, maxVals):
            candidate.append(trial.suggest_uniform(paramLabel, minVal, maxVal))

        # remember pids and jobids in a list
        pids = []
        jobids = {}

        # create a job for the candidate
        candidate_index = 0

        sleep(sleepInterval)  # required for slurm

        # name and path
        jobName = "trial_" + str(ngen)
        jobPath = genFolderPath + '/' + jobName

        # set initial cfg initCfg
        if len(self.initCfg) > 0:
            for paramLabel, paramVal in self.initCfg.items():
                self.setCfgNestedParam(paramLabel, paramVal)

        # modify cfg instance with candidate values
        #print(paramLabels, candidate)
        for label, value in zip(paramLabels, candidate):
            print('set %s=%s' % (label, value))
            self.setCfgNestedParam(label, value)

        #self.setCfgNestedParam("filename", jobPath)
        self.cfg.simLabel = jobName
        self.cfg.saveFolder = genFolderPath

        # save cfg instance to file
        cfgSavePath = jobPath + '_cfg.json'
        self.cfg.save(cfgSavePath)

        if type=='mpi_bulletin':
            # ----------------------------------------------------------------------
            # MPI master-slaves
            # ----------------------------------------------------------------------
            pc.submit(runJob, script, cfgSavePath, netParamsSavePath, jobPath)
            print('-'*80)

        else:
            # ----------------------------------------------------------------------
            # MPI job commnand
            # ----------------------------------------------------------------------
            command = '%s -np %d nrniv -python -mpi %s simConfig=%s netParams=%s ' % (mpiCommand, numproc, script, cfgSavePath, netParamsSavePath)

            # ----------------------------------------------------------------------
            # run on local machine with <nodes*coresPerNode> cores
            # ----------------------------------------------------------------------
            if type=='mpi_direct':
                executer = '/bin/bash'
                jobString = bashTemplate('mpi_direct') %(custom, folder, command)

            # ----------------------------------------------------------------------
            # run on HPC through slurm
            # ----------------------------------------------------------------------
            elif type=='hpc_slurm':
                executer = 'sbatch'
                res = '#SBATCH --res=%s' % (reservation) if reservation else ''
                jobString = bashTemplate('hpc_slurm') % (jobName, allocation, walltime, nodes, coresPerNode, jobPath, jobPath, email, res, custom, folder, command)

            # ----------------------------------------------------------------------
            # run on HPC through PBS
            # ----------------------------------------------------------------------
            elif type=='hpc_torque':
                executer = 'qsub'
                queueName = args.get('queueName', 'default')
                nodesppn = 'nodes=%d:ppn=%d' % (nodes, coresPerNode)
                jobString = bashTemplate('hpc_torque') % (jobName, walltime, queueName, nodesppn, jobPath, jobPath, custom, command)

            # ----------------------------------------------------------------------
            # save job and run
            # ----------------------------------------------------------------------
            print('Submitting job ', jobName)
            print(jobString)
            print('-'*80)
            # save file
            batchfile = '%s.sbatch' % (jobPath)
            with open(batchfile, 'w') as text_file:
                text_file.write("%s" % jobString)

            if type == 'mpi_direct':
                with open(jobPath+'.run', 'a+') as outf, open(jobPath+'.err', 'w') as errf:
                    pids.append(Popen([executer, batchfile], stdout=outf, stderr=errf, preexec_fn=os.setsid).pid)
            else:
                with open(jobPath+'.jobid', 'w') as outf, open(jobPath+'.err', 'w') as errf:
                    pids.append(Popen([executer, batchfile], stdout=outf, stderr=errf, preexec_fn=os.setsid).pid)

            #proc = Popen(command.split([executer, batchfile]), stdout=PIPE, stderr=PIPE)
            sleep(0.1)
            #read = proc.stdout.read()

            if type == 'mpi_direct':
                with open('./pids.pid', 'a') as file:
                    file.write(str(pids))
            else:
                with open(jobPath+'.jobid', 'r') as outf:
                    read=outf.readline()
                print(read)
                if len(read) > 0:
                    jobid = int(read.split()[-1])
                    jobids[candidate_index] = jobid
                print('jobids', jobids)
        total_jobs += 1
        sleep(0.1)

        # ----------------------------------------------------------------------
        # gather data and compute fitness
        # ----------------------------------------------------------------------
        if type == 'mpi_bulletin':
            # wait for pc bulletin board jobs to finish
            try:
                while pc.working():
                    sleep(1)
                #pc.done()
            except:
                pass

        num_iters = 0
        jobs_completed = 0
        fitness = [None]  # just 1 candidate
        # print outfilestem
        print("Waiting for jobs from generation %d/%d ..." %(ngen, args.get('maxiters')))
        # print "PID's: %r" %(pids)
        # start fitness calculation
        while jobs_completed < total_jobs:
            unfinished = [i for i, x in enumerate(fitness) if x is None ]
            for candidate_index in unfinished:
                #try: # load simData and evaluate fitness
                jobNamePath = genFolderPath + "/trial_" + str(ngen)
                if os.path.isfile(jobNamePath+'.json'):
                    with open('%s.json'% (jobNamePath)) as file:
                        simData = json.load(file)['simData']
                    fitness[candidate_index] = fitnessFunc(simData, **fitnessFuncArgs)
                    jobs_completed += 1
                    print('  Candidate %d fitness = %.1f' % (candidate_index, fitness[candidate_index]))
                # except Exception as e:
                #     err = "There was an exception evaluating candidate %d:"%(candidate_index)
                #     print(("%s \n %s"%(err,e)))
            num_iters += 1
            print('completed: %d' %(jobs_completed))
            if num_iters >= args.get('maxiter_wait', 5000):
                print("Max iterations reached, the %d unfinished jobs will be canceled and set to default fitness" % (len(unfinished)))
                for canditade_index in unfinished:
                    fitness[canditade_index] = maxFitness # rerun those that didn't complete;
                    jobs_completed += 1
                    try:
                        if 'scancelUser' in kwargs:
                            os.system('scancel -u %s'%(kwargs['scancelUser']))
                        else:
                            os.system('scancel %d' % (jobids[candidate_index]))  # terminate unfinished job (resubmitted jobs not terminated!)
                    except:
                        pass
            sleep(args.get('time_sleep', 1))

        # kill all processes
        if type == 'mpi_bulletin':
            try:
                with open("./pids.pid", 'r') as file: # read pids for mpi_bulletin
                    pids = [int(i) for i in file.read().split(' ')[:-1]]

                with open("./pids.pid", 'w') as file: # delete content
                    pass
                for pid in pids:
                    try:
                        os.killpg(os.getpgid(pid), signal.SIGTERM)
                    except:
                        pass
            except:
                pass

        elif type == 'mpi_direct':

            import psutil

            PROCNAME = "nrniv"

            for proc in psutil.process_iter():
                # check whether the process name matches
                try:
                    if proc.name() == PROCNAME:
                        proc.kill()
                except:
                    pass

        # don't want to to this for hpcs since jobs are running on compute nodes not master

        print("-" * 80)
        print("  Completed a generation  ")
        print("-" * 80)

        return fitness[0] # single candidate for now

    # -------------------------------------------------------------------------------
    # Optuna optimization: Main code
    # -------------------------------------------------------------------------------
    import os
    from time import sleep
    from mpi4py import MPI

    comm = MPI.COMM_WORLD
    size = comm.Get_size()
    rank = comm.Get_rank()

    # create main sim directory and save scripts
    self.saveScripts()

    global ngen
    ngen = -1

    # gather **kwargs

    args = {}
    args['popsize'] = self.optimCfg.get('popsize', 1)
    args['minVals'] = [x['values'][0] for x in self.params]
    args['maxVals'] = [x['values'][1] for x in self.params]
    args['cfg'] = self.cfg  # include here args/params to pass to evaluator function
    args['paramLabels'] = [x['label'] for x in self.params]
    args['netParamsSavePath'] = self.saveFolder + '/' + self.batchLabel + '_netParams.py'
    args['maxiters'] = self.optimCfg['maxiters'] if 'maxiters' in self.optimCfg else 1000
    args['maxtime'] = self.optimCfg['maxtime'] if 'maxtime' in self.optimCfg else None
    args['fitnessFunc'] = self.optimCfg['fitnessFunc']
    args['fitnessFuncArgs'] = self.optimCfg['fitnessFuncArgs']
    args['maxiter_wait'] = self.optimCfg['maxiter_wait']
    args['time_sleep'] = self.optimCfg['time_sleep']
    args['maxFitness'] = self.optimCfg.get('maxFitness', 1000)

    for key, value in self.optimCfg.items():
        args[key] = value

    for key, value in self.runCfg.items():
        args[key] = value

    # if using pc bulletin board, initialize all workers
    if self.runCfg.get('type', None) == 'mpi_bulletin':
        for iworker in range(int(pc.nhost())):
            pc.runworker()

    # -------------------------------------------------------------------------------
    # Run algorithm
    # -------------------------------------------------------------------------------

    sleep(rank) # each process wiats a different time to avoid saturating sqlite database
    study = optuna.create_study(study_name=self.batchLabel, storage='sqlite:///%s/%s_storage.db' % (self.saveFolder, self.batchLabel), load_if_exists=True)
    study.optimize(lambda trial: objective(trial, args), n_trials=args['maxiters'], timeout=args['maxtime'])

    # print best and finish
    if rank == size-1:
        df = study.trials_dataframe(attrs=('number', 'value', 'params', 'state'))
        importance = optuna.importance.get_param_importances(study=study)

        print('\nBest trial: ', study.best_trial)
        print('\nParameter importance: ', dict(importance))

        print('\nBest Solution with fitness = %.4g: \n' % (study.best_value), study.best_params)

        print('\nSaving to output.pkl...\n')
        output = {'study': study, 'df': df, 'importance': importance}
        with open('%s/%s_output.pkl' % (self.saveFolder, self.batchLabel), 'wb') as f:
            pickle.dump(output, f)

        sleep(1)

        print("-" * 80)
        print("   Completed Optuna parameter optimization   ")
        print("-" * 80)

    sys.exit()
<commit_msg>add exception for mpi4py<commit_after>"""
batch/evol.py

Code for adaptive stochastic descent optimizaiton method (Kerr et al. 2018; PLoS ONE)

Contributors: salvadordura@gmail.com
"""

from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import

from builtins import zip

from builtins import range
from builtins import open
from builtins import str
from future import standard_library
standard_library.install_aliases()

# required to make json saving work in Python 2/3
try:
    to_unicode = unicode
except NameError:
    to_unicode = str

import imp
import json
import logging
import datetime
import os
import signal
import glob
from copy import copy
from random import Random
from time import sleep, time
from itertools import product
from subprocess import Popen, PIPE
import importlib, types
import numpy.random as nr
import numpy as np
import pickle

from neuron import h
from netpyne import sim, specs
import optuna
from .utils import createFolder
from .utils import bashTemplate
from .utils import dcp, sigfig

pc = h.ParallelContext() # use bulletin board master/slave

# -------------------------------------------------------------------------------
# Optuna optimization
# -------------------------------------------------------------------------------

# func needs to be outside of class
def runJob(script, cfgSavePath, netParamsSavePath, simDataPath):
    import os
    print('\nJob in rank id: ',pc.id())
    command = 'nrniv %s simConfig=%s netParams=%s' % (script, cfgSavePath, netParamsSavePath)
    print(command)

    with open(simDataPath+'.run', 'w') as outf, open(simDataPath+'.err', 'w') as errf:
        pid = Popen(command.split(' '), stdout=outf, stderr=errf, preexec_fn=os.setsid).pid

    with open('./pids.pid', 'a') as file:
        file.write(str(pid) + ' ')

def optunaOptim(self, pc):
    import sys

    # -------------------------------------------------------------------------------
    # Optuna optimization: Parallel evaluation
    # -------------------------------------------------------------------------------
    def objective(trial, args):

        import os

        ngen = trial.number
        total_jobs = 0

        # options slurm, mpi
        type = args.get('type', 'mpi_direct')

        # params
        paramLabels = args.get('paramLabels', [])
        minVals = args.get('minVals', [])
        maxVals = args.get('maxVals', [])

        # paths to required scripts
        script = args.get('script', 'init.py')
        netParamsSavePath =  args.get('netParamsSavePath')
        genFolderPath = self.saveFolder + '/trial_' + str(ngen)

        # mpi command setup
        nodes = args.get('nodes', 1)
        coresPerNode = args.get('coresPerNode', 1)
        mpiCommand = args.get('mpiCommand', 'ibrun')
        numproc = nodes*coresPerNode

        # slurm setup
        custom = args.get('custom', '')
        folder = args.get('folder', '.')
        email = args.get('email', 'a@b.c')
        walltime = args.get('walltime', '00:01:00')
        reservation = args.get('reservation', None)
        allocation = args.get('allocation', 'csd403') # NSG account

        # fitness function
        fitnessFunc = args.get('fitnessFunc')
        fitnessFuncArgs = args.get('fitnessFuncArgs')
        maxFitness = args.get('maxFitness')

        # read params or set defaults
        sleepInterval = args.get('sleepInterval', 0.2)

        # create folder if it does not exist
        createFolder(genFolderPath)

        # --------------------------------------
        # generate param values for optuna trial
        candidate = []
        for paramLabel, minVal, maxVal in zip(paramLabels, minVals, maxVals):
            candidate.append(trial.suggest_uniform(paramLabel, minVal, maxVal))

        # remember pids and jobids in a list
        pids = []
        jobids = {}

        # create a job for the candidate
        candidate_index = 0

        sleep(sleepInterval)  # required for slurm

        # name and path
        jobName = "trial_" + str(ngen)
        jobPath = genFolderPath + '/' + jobName

        # set initial cfg initCfg
        if len(self.initCfg) > 0:
            for paramLabel, paramVal in self.initCfg.items():
                self.setCfgNestedParam(paramLabel, paramVal)

        # modify cfg instance with candidate values
        #print(paramLabels, candidate)
        for label, value in zip(paramLabels, candidate):
            print('set %s=%s' % (label, value))
            self.setCfgNestedParam(label, value)

        #self.setCfgNestedParam("filename", jobPath)
        self.cfg.simLabel = jobName
        self.cfg.saveFolder = genFolderPath

        # save cfg instance to file
        cfgSavePath = jobPath + '_cfg.json'
        self.cfg.save(cfgSavePath)

        if type=='mpi_bulletin':
            # ----------------------------------------------------------------------
            # MPI master-slaves
            # ----------------------------------------------------------------------
            pc.submit(runJob, script, cfgSavePath, netParamsSavePath, jobPath)
            print('-'*80)

        else:
            # ----------------------------------------------------------------------
            # MPI job commnand
            # ----------------------------------------------------------------------
            command = '%s -np %d nrniv -python -mpi %s simConfig=%s netParams=%s ' % (mpiCommand, numproc, script, cfgSavePath, netParamsSavePath)

            # ----------------------------------------------------------------------
            # run on local machine with <nodes*coresPerNode> cores
            # ----------------------------------------------------------------------
            if type=='mpi_direct':
                executer = '/bin/bash'
                jobString = bashTemplate('mpi_direct') %(custom, folder, command)

            # ----------------------------------------------------------------------
            # run on HPC through slurm
            # ----------------------------------------------------------------------
            elif type=='hpc_slurm':
                executer = 'sbatch'
                res = '#SBATCH --res=%s' % (reservation) if reservation else ''
                jobString = bashTemplate('hpc_slurm') % (jobName, allocation, walltime, nodes, coresPerNode, jobPath, jobPath, email, res, custom, folder, command)

            # ----------------------------------------------------------------------
            # run on HPC through PBS
            # ----------------------------------------------------------------------
            elif type=='hpc_torque':
                executer = 'qsub'
                queueName = args.get('queueName', 'default')
                nodesppn = 'nodes=%d:ppn=%d' % (nodes, coresPerNode)
                jobString = bashTemplate('hpc_torque') % (jobName, walltime, queueName, nodesppn, jobPath, jobPath, custom, command)

            # ----------------------------------------------------------------------
            # save job and run
            # ----------------------------------------------------------------------
            print('Submitting job ', jobName)
            print(jobString)
            print('-'*80)
            # save file
            batchfile = '%s.sbatch' % (jobPath)
            with open(batchfile, 'w') as text_file:
                text_file.write("%s" % jobString)

            if type == 'mpi_direct':
                with open(jobPath+'.run', 'a+') as outf, open(jobPath+'.err', 'w') as errf:
                    pids.append(Popen([executer, batchfile], stdout=outf, stderr=errf, preexec_fn=os.setsid).pid)
            else:
                with open(jobPath+'.jobid', 'w') as outf, open(jobPath+'.err', 'w') as errf:
                    pids.append(Popen([executer, batchfile], stdout=outf, stderr=errf, preexec_fn=os.setsid).pid)

            #proc = Popen(command.split([executer, batchfile]), stdout=PIPE, stderr=PIPE)
            sleep(0.1)
            #read = proc.stdout.read()

            if type == 'mpi_direct':
                with open('./pids.pid', 'a') as file:
                    file.write(str(pids))
            else:
                with open(jobPath+'.jobid', 'r') as outf:
                    read=outf.readline()
                print(read)
                if len(read) > 0:
                    jobid = int(read.split()[-1])
                    jobids[candidate_index] = jobid
                print('jobids', jobids)
        total_jobs += 1
        sleep(0.1)

        # ----------------------------------------------------------------------
        # gather data and compute fitness
        # ----------------------------------------------------------------------
        if type == 'mpi_bulletin':
            # wait for pc bulletin board jobs to finish
            try:
                while pc.working():
                    sleep(1)
                #pc.done()
            except:
                pass

        num_iters = 0
        jobs_completed = 0
        fitness = [None]  # just 1 candidate
        # print outfilestem
        print("Waiting for jobs from generation %d/%d ..." %(ngen, args.get('maxiters')))
        # print "PID's: %r" %(pids)
        # start fitness calculation
        while jobs_completed < total_jobs:
            unfinished = [i for i, x in enumerate(fitness) if x is None ]
            for candidate_index in unfinished:
                #try: # load simData and evaluate fitness
                jobNamePath = genFolderPath + "/trial_" + str(ngen)
                if os.path.isfile(jobNamePath+'.json'):
                    with open('%s.json'% (jobNamePath)) as file:
                        simData = json.load(file)['simData']
                    fitness[candidate_index] = fitnessFunc(simData, **fitnessFuncArgs)
                    jobs_completed += 1
                    print('  Candidate %d fitness = %.1f' % (candidate_index, fitness[candidate_index]))
                # except Exception as e:
                #     err = "There was an exception evaluating candidate %d:"%(candidate_index)
                #     print(("%s \n %s"%(err,e)))
            num_iters += 1
            print('completed: %d' %(jobs_completed))
            if num_iters >= args.get('maxiter_wait', 5000):
                print("Max iterations reached, the %d unfinished jobs will be canceled and set to default fitness" % (len(unfinished)))
                for canditade_index in unfinished:
                    fitness[canditade_index] = maxFitness # rerun those that didn't complete;
                    jobs_completed += 1
                    try:
                        if 'scancelUser' in kwargs:
                            os.system('scancel -u %s'%(kwargs['scancelUser']))
                        else:
                            os.system('scancel %d' % (jobids[candidate_index]))  # terminate unfinished job (resubmitted jobs not terminated!)
                    except:
                        pass
            sleep(args.get('time_sleep', 1))

        # kill all processes
        if type == 'mpi_bulletin':
            try:
                with open("./pids.pid", 'r') as file: # read pids for mpi_bulletin
                    pids = [int(i) for i in file.read().split(' ')[:-1]]

                with open("./pids.pid", 'w') as file: # delete content
                    pass
                for pid in pids:
                    try:
                        os.killpg(os.getpgid(pid), signal.SIGTERM)
                    except:
                        pass
            except:
                pass

        elif type == 'mpi_direct':

            import psutil

            PROCNAME = "nrniv"

            for proc in psutil.process_iter():
                # check whether the process name matches
                try:
                    if proc.name() == PROCNAME:
                        proc.kill()
                except:
                    pass

        # don't want to to this for hpcs since jobs are running on compute nodes not master

        print("-" * 80)
        print("  Completed a generation  ")
        print("-" * 80)

        return fitness[0] # single candidate for now

    # -------------------------------------------------------------------------------
    # Optuna optimization: Main code
    # -------------------------------------------------------------------------------
    import os
    from time import sleep

    try:
        from mpi4py import MPI

        comm = MPI.COMM_WORLD
        size = comm.Get_size()
        rank = comm.Get_rank()
    except:
        size = 1
        rank = 0

    # create main sim directory and save scripts
    self.saveScripts()

    global ngen
    ngen = -1

    # gather **kwargs

    args = {}
    args['popsize'] = self.optimCfg.get('popsize', 1)
    args['minVals'] = [x['values'][0] for x in self.params]
    args['maxVals'] = [x['values'][1] for x in self.params]
    args['cfg'] = self.cfg  # include here args/params to pass to evaluator function
    args['paramLabels'] = [x['label'] for x in self.params]
    args['netParamsSavePath'] = self.saveFolder + '/' + self.batchLabel + '_netParams.py'
    args['maxiters'] = self.optimCfg['maxiters'] if 'maxiters' in self.optimCfg else 1000
    args['maxtime'] = self.optimCfg['maxtime'] if 'maxtime' in self.optimCfg else None
    args['fitnessFunc'] = self.optimCfg['fitnessFunc']
    args['fitnessFuncArgs'] = self.optimCfg['fitnessFuncArgs']
    args['maxiter_wait'] = self.optimCfg['maxiter_wait']
    args['time_sleep'] = self.optimCfg['time_sleep']
    args['maxFitness'] = self.optimCfg.get('maxFitness', 1000)

    for key, value in self.optimCfg.items():
        args[key] = value

    for key, value in self.runCfg.items():
        args[key] = value

    # if using pc bulletin board, initialize all workers
    if self.runCfg.get('type', None) == 'mpi_bulletin':
        for iworker in range(int(pc.nhost())):
            pc.runworker()

    # -------------------------------------------------------------------------------
    # Run algorithm
    # -------------------------------------------------------------------------------

    sleep(rank) # each process wiats a different time to avoid saturating sqlite database
    study = optuna.create_study(study_name=self.batchLabel, storage='sqlite:///%s/%s_storage.db' % (self.saveFolder, self.batchLabel), load_if_exists=True)
    study.optimize(lambda trial: objective(trial, args), n_trials=args['maxiters'], timeout=args['maxtime'])

    # print best and finish
    if rank == size-1:
        df = study.trials_dataframe(attrs=('number', 'value', 'params', 'state'))
        importance = optuna.importance.get_param_importances(study=study)

        print('\nBest trial: ', study.best_trial)
        print('\nParameter importance: ', dict(importance))

        print('\nBest Solution with fitness = %.4g: \n' % (study.best_value), study.best_params)

        print('\nSaving to output.pkl...\n')
        output = {'study': study, 'df': df, 'importance': importance}
        with open('%s/%s_output.pkl' % (self.saveFolder, self.batchLabel), 'wb') as f:
            pickle.dump(output, f)

        sleep(1)

        print("-" * 80)
        print("   Completed Optuna parameter optimization   ")
        print("-" * 80)

    sys.exit()
<|endoftext|>


<commit_before>from __future__ import division, print_function
import numpy as np
from scipy.ndimage import gaussian_filter

from .data import Trace, PatchClampRecording

def detect_evoked_spike(data, pulse_edges, **kwds):
    """Return a dict describing an evoked spike in a patch clamp recording, or None if no spike is detected.

    Parameters
    ==========
    data : PatchClampRecording
        The recorded patch clamp data. The recording should be made with a brief pulse intended
        to evoke a single spike with short latency.
    pulse_edges : (int, int)
        The start and end indices of the stimulation pulse.
    """
    trace = data['primary']
    if data.clamp_mode == 'vc':
        return detect_vc_evoked_spike(trace, pulse_edges, **kwds)
    elif data.clamp_mode == 'ic':
        return detect_ic_evoked_spike(trace, pulse_edges, **kwds)
    else:
        raise ValueError("Unsupported clamp mode %s" % trace.clamp_mode)

def detect_ic_evoked_spike(trace, pulse_edges, threshold=-10e-3, duration=3e-3):
    assert trace.data.ndim == 1
    pulse_edges = tuple(map(int, pulse_edges))  # make sure pulse_edges is (int, int)

    dt = trace.dt
    w = int(duration / dt)
    pstart, pstop = pulse_edges
    chunk = trace.data[pstart:pstart+w]

    peak_ind = np.argmax(chunk)
    peak_val = chunk[peak_ind]
    if peak_val < threshold:
        # trace did not cross threshold; no spike here
        return None
    if peak_ind < 3:
        # peak find failed--this can happen when all samples in
        # the chunk have the same value
        return None

    dvdt = np.diff(chunk[:peak_ind])
    rise_ind = np.argmax(dvdt)
    max_dvdt = dvdt[rise_ind]

    return {'peak_index': peak_ind + pstart, 'rise_index': rise_ind + pstart,
            'peak_val': peak_val, 'max_dvdt': max_dvdt}

def detect_vc_evoked_spike(trace, pulse_edges, sigma=20e-6, delay=150e-6, threshold=50e-12):
    """Return a dict describing an evoked spike in a patch clamp recording, or None if no spike is detected.

    This function assumes that a square voltage pulse is used to evoke an unclamped spike
    in a voltage clamp recording, and that the peak of the unclamped spike occurs *during*
    the stimulation pulse.

    Parameters
    ==========
    trace : Trace instance
        The recorded patch clamp data. The recording should be made with a brief pulse intended
        to evoke a single spike with short latency.
    pulse_edges : (int, int)
        The start and end indices of the stimulation pulse.
    sigma : float
        Time constant (sec) for a gaussian filter used to smooth the trace before
        peak detection. default is 20 us.
    delay : float
        Time (sec) after the onset of the stimulation pulse to begin searching for the
        peak of the unclamped spike. This is used to exclude artifacts that can
        appear immediately after the pulse onset. Default is 150 us.
    threshold : float
        Threshold (amps) for detection of the unclamped spike peak (see Notes). Default
        is 50 pA.

    Notes
    =====

    This is a simple unclamped spike detection algorithm that compares the minimum
    pipette current value during the stimulation pulse to the final value before
    pulse offset. An unclamped spike is detected if the minimum value is less
    than the final value, and the difference is greater than the specified
    threshold. The rising phase of the action potential is determined by searching
    for a local minimum in the derivative of the pipette current immediately before
    the detected peak.

    This algorithm generally performs well for spike detection, with few false
    positives or negatives. However, the max dv/dt measurement may be invalid for
    very small unclamped spikes, making the true latency of the spike difficult
    to measure.
    """
    if not isinstance(trace, Trace):
        raise TypeError("data must be PatchClampRecording or Trace instance.")

    assert trace.ndim == 1
    pulse_edges = tuple(map(int, pulse_edges))  # make sure pulse_edges is (int, int)

    dt = trace.dt

    # select just the portion of the trace that contains the pulse, excluding a short
    # delay after the pulse onset
    pstart = pulse_edges[0] + int(delay / dt)
    pstop = pulse_edges[1]

    # find the location of the minimum value during the pulse
    smooth = gaussian_filter(trace.data[pstart:pstop], int(sigma/dt))
    if len(smooth) == 0:
        raise ValueError("Invalid pulse indices [%d->%d:%d] for data (%d)" % (pulse_edges[0], pstart, pstop, len(trace.data)))
    peak_ind = np.argmin(smooth)
    if peak_ind == 0 or peak_ind == len(smooth)-1:
        # no local minimum found within pulse edges
        return None

    # a spike is detected only if the peak is at least 50pA less than the final value before pulse offset
    peak_diff = smooth[-1] - smooth[peak_ind]
    if peak_diff > threshold:

        # Walk backward to the point of max dv/dt
        dv = np.diff(smooth)
        rstart = max(0, peak_ind - int(1e-3/dt))  # don't search for rising phase more than 1ms before peak
        rise_ind = np.argmin(dv[rstart:peak_ind])
        max_dvdt = dv[rise_ind] / dt
        rise_ind += pstart + rstart

        return {'peak_index': peak_ind + pstart, 'rise_index': rise_ind,
                'peak_diff': peak_diff, 'max_dvdt': max_dvdt}
    else:
        return None
<commit_msg>change dvdt detection to look back 1ms from spike peak<commit_after>from __future__ import division, print_function
import numpy as np
from scipy.ndimage import gaussian_filter

from .data import Trace, PatchClampRecording

def detect_evoked_spike(data, pulse_edges, **kwds):
    """Return a dict describing an evoked spike in a patch clamp recording, or None if no spike is detected.

    Parameters
    ==========
    data : PatchClampRecording
        The recorded patch clamp data. The recording should be made with a brief pulse intended
        to evoke a single spike with short latency.
    pulse_edges : (int, int)
        The start and end indices of the stimulation pulse.
    """
    trace = data['primary']
    if data.clamp_mode == 'vc':
        return detect_vc_evoked_spike(trace, pulse_edges, **kwds)
    elif data.clamp_mode == 'ic':
        return detect_ic_evoked_spike(trace, pulse_edges, **kwds)
    else:
        raise ValueError("Unsupported clamp mode %s" % trace.clamp_mode)

def detect_ic_evoked_spike(trace, pulse_edges, threshold=-10e-3, duration=3e-3, max_dvdt_search=1e-3):
    assert trace.data.ndim == 1
    pulse_edges = tuple(map(int, pulse_edges))  # make sure pulse_edges is (int, int)

    dt = trace.dt
    w = int(duration / dt)
    pstart, pstop = pulse_edges
    chunk = trace.data[pstart:pstart+w]

    peak_ind = np.argmax(chunk)
    peak_val = chunk[peak_ind]
    if peak_val < threshold:
        # trace did not cross threshold; no spike here
        return None
    if peak_ind < 3:
        # peak find failed--this can happen when all samples in
        # the chunk have the same value
        return None

    # In some rare cases, the stimulus causes a very high dvdt at the beginning of the pulse.
    # We catch this by only searching for max dvdt within a certain window of the peak.
    maxdvdt_start_index = max(0, peak_ind - int(max_dvdt_search / dt))
    dvdt = np.diff(chunk[maxdvdt_start_index:peak_ind])
    rise_ind = np.argmax(dvdt)
    max_dvdt = dvdt[rise_ind]

    return {'peak_index': peak_ind + pstart, 'rise_index': rise_ind + pstart + maxdvdt_start_index,
            'peak_val': peak_val, 'max_dvdt': max_dvdt}

def detect_vc_evoked_spike(trace, pulse_edges, sigma=20e-6, delay=150e-6, threshold=50e-12):
    """Return a dict describing an evoked spike in a patch clamp recording, or None if no spike is detected.

    This function assumes that a square voltage pulse is used to evoke an unclamped spike
    in a voltage clamp recording, and that the peak of the unclamped spike occurs *during*
    the stimulation pulse.

    Parameters
    ==========
    trace : Trace instance
        The recorded patch clamp data. The recording should be made with a brief pulse intended
        to evoke a single spike with short latency.
    pulse_edges : (int, int)
        The start and end indices of the stimulation pulse.
    sigma : float
        Time constant (sec) for a gaussian filter used to smooth the trace before
        peak detection. default is 20 us.
    delay : float
        Time (sec) after the onset of the stimulation pulse to begin searching for the
        peak of the unclamped spike. This is used to exclude artifacts that can
        appear immediately after the pulse onset. Default is 150 us.
    threshold : float
        Threshold (amps) for detection of the unclamped spike peak (see Notes). Default
        is 50 pA.

    Notes
    =====

    This is a simple unclamped spike detection algorithm that compares the minimum
    pipette current value during the stimulation pulse to the final value before
    pulse offset. An unclamped spike is detected if the minimum value is less
    than the final value, and the difference is greater than the specified
    threshold. The rising phase of the action potential is determined by searching
    for a local minimum in the derivative of the pipette current immediately before
    the detected peak.

    This algorithm generally performs well for spike detection, with few false
    positives or negatives. However, the max dv/dt measurement may be invalid for
    very small unclamped spikes, making the true latency of the spike difficult
    to measure.
    """
    if not isinstance(trace, Trace):
        raise TypeError("data must be PatchClampRecording or Trace instance.")

    assert trace.ndim == 1
    pulse_edges = tuple(map(int, pulse_edges))  # make sure pulse_edges is (int, int)

    dt = trace.dt

    # select just the portion of the trace that contains the pulse, excluding a short
    # delay after the pulse onset
    pstart = pulse_edges[0] + int(delay / dt)
    pstop = pulse_edges[1]

    # find the location of the minimum value during the pulse
    smooth = gaussian_filter(trace.data[pstart:pstop], int(sigma/dt))
    if len(smooth) == 0:
        raise ValueError("Invalid pulse indices [%d->%d:%d] for data (%d)" % (pulse_edges[0], pstart, pstop, len(trace.data)))
    peak_ind = np.argmin(smooth)
    if peak_ind == 0 or peak_ind == len(smooth)-1:
        # no local minimum found within pulse edges
        return None

    # a spike is detected only if the peak is at least 50pA less than the final value before pulse offset
    peak_diff = smooth[-1] - smooth[peak_ind]
    if peak_diff > threshold:

        # Walk backward to the point of max dv/dt
        dv = np.diff(smooth)
        rstart = max(0, peak_ind - int(1e-3/dt))  # don't search for rising phase more than 1ms before peak
        rise_ind = np.argmin(dv[rstart:peak_ind])
        max_dvdt = dv[rise_ind] / dt
        rise_ind += pstart + rstart

        return {'peak_index': peak_ind + pstart, 'rise_index': rise_ind,
                'peak_diff': peak_diff, 'max_dvdt': max_dvdt}
    else:
        return None
<|endoftext|>


<commit_before>import operator
MIN_FLOAT=-3.14e100

def get_top_states(t_state_v,K=4):
    items = t_state_v.items()
    topK= sorted(items,key=operator.itemgetter(1),reverse=True)[:K]
    return [x[0] for x in topK]

def viterbi(obs, states, start_p, trans_p, emit_p):
    V = [{}] #tabular
    mem_path = [{}]
    all_states = trans_p.keys()
    for y in states.get(obs[0],all_states): #init
        V[0][y] = start_p[y] + emit_p[y].get(obs[0],MIN_FLOAT)
        mem_path[0][y] = ''
    for t in range(1,len(obs)):
        V.append({})
        mem_path.append({})
        prev_states = get_top_states(V[t-1])
        prev_states =[ x for x in mem_path[t-1].keys() if len(trans_p[x])>0 ]

        prev_states_expect_next = set( (y  for x in prev_states for y in trans_p[x].keys() ) )
        obs_states = states.get(obs[t],all_states)
        obs_states = set(obs_states) &  set(prev_states_expect_next)

        if len(obs_states)==0: obs_states = all_states
        for y in obs_states:
            (prob,state ) = max([(V[t-1][y0] + trans_p[y0].get(y,MIN_FLOAT) + emit_p[y].get(obs[t],MIN_FLOAT) ,y0) for y0 in prev_states])
            V[t][y] =prob
            mem_path[t][y] = state

    last = [(V[-1][y], y) for y in mem_path[-1].keys() ]
    #if len(last)==0:
        #print obs
    (prob, state) = max(last)

    route = [None] * len(obs)
    i = len(obs)-1
    while i>=0:
        route[i] = state
        state = mem_path[i][state]
        i-=1
    return (prob, route)<commit_msg>fix a bug in issue #111<commit_after>import operator
MIN_FLOAT=-3.14e100

def get_top_states(t_state_v,K=4):
    items = t_state_v.items()
    topK= sorted(items,key=operator.itemgetter(1),reverse=True)[:K]
    return [x[0] for x in topK]

def viterbi(obs, states, start_p, trans_p, emit_p):
    V = [{}] #tabular
    mem_path = [{}]
    all_states = trans_p.keys()
    for y in states.get(obs[0],all_states): #init
        V[0][y] = start_p[y] + emit_p[y].get(obs[0],MIN_FLOAT)
        mem_path[0][y] = ''
    for t in range(1,len(obs)):
        V.append({})
        mem_path.append({})
        #prev_states = get_top_states(V[t-1])
        prev_states =[ x for x in mem_path[t-1].keys() if len(trans_p[x])>0 ]

        prev_states_expect_next = set( (y  for x in prev_states for y in trans_p[x].keys() ) )
        obs_states = states.get(obs[t],all_states)
        obs_states = set(obs_states) &  set(prev_states_expect_next)

        if len(obs_states)==0: obs_states = all_states
        for y in obs_states:
            (prob,state ) = max([(V[t-1][y0] + trans_p[y0].get(y,MIN_FLOAT) + emit_p[y].get(obs[t],MIN_FLOAT) ,y0) for y0 in prev_states])
            V[t][y] =prob
            mem_path[t][y] = state

    last = [(V[-1][y], y) for y in mem_path[-1].keys() ]
    #if len(last)==0:
        #print obs
    (prob, state) = max(last)

    route = [None] * len(obs)
    i = len(obs)-1
    while i>=0:
        route[i] = state
        state = mem_path[i][state]
        i-=1
    return (prob, route)<|endoftext|>


<commit_before>from functools import reduce
from django.conf import settings  # type: ignore
from django.contrib.auth.models import User  # type: ignore
from django.db.models import ManyToManyField, ForeignKey, \
    URLField, IntegerField, CharField, CASCADE, TextField, \
    DateTimeField, Model, BooleanField, Q  # type: ignore
from django.utils.translation import ugettext_lazy as _  # type: ignore
from nextcloudappstore.core.versioning import pad_min_version, \
    pad_max_inc_version
from parler.models import TranslatedFields, TranslatableModel, \
    TranslatableManager  # type: ignore
from semantic_version import Version, Spec

class AppManager(TranslatableManager):
    def search(self, terms, lang):
        queryset = self.get_queryset().active_translations(lang).language(
            lang).distinct()
        predicates = map(lambda t: (Q(translations__name__icontains=t) |
                                    Q(translations__summary__icontains=t) |
                                    Q(translations__description__icontains=t)),
                         terms)
        query = reduce(lambda x, y: x & y, predicates, Q())
        return queryset.filter(query)

    def get_compatible(self, platform_version, inclusive=False):
        apps = App.objects.prefetch_related('translations', 'screenshots',
                                            'releases', 'releases__databases',
                                            'releases__php_extensions').all()

        def app_filter(app):
            for release in app.releases.all():
                if release.is_compatible(platform_version, inclusive):
                    return True
            return False

        return list(filter(app_filter, apps))

class App(TranslatableModel):
    objects = AppManager()
    id = CharField(max_length=128, unique=True, primary_key=True,
                   verbose_name=_('Id'),
                   help_text=_('app id, identical to folder name'))
    categories = ManyToManyField('Category', verbose_name=_('Category'))
    translations = TranslatedFields(
        name=CharField(max_length=128, verbose_name=_('Name'),
                       help_text=_('Rendered app name for users')),
        summary=CharField(max_length=128, verbose_name=_('Summary'),
                          help_text=_(
                              'Short text describing the app\'s purpose')),
        description=TextField(verbose_name=_('Description'), help_text=_(
            'Will be rendered as Markdown'))
    )
    # resources
    user_docs = URLField(max_length=256, blank=True,
                         verbose_name=_('User documentation url'))
    admin_docs = URLField(max_length=256, blank=True,
                          verbose_name=_('Admin documentation url'))
    developer_docs = URLField(max_length=256, blank=True,
                              verbose_name=_('Developer documentation url'))
    issue_tracker = URLField(max_length=256, blank=True,
                             verbose_name=_('Issue tracker url'))
    website = URLField(max_length=256, blank=True, verbose_name=_('Homepage'))
    created = DateTimeField(auto_now_add=True, editable=False,
                            verbose_name=_('Created at'))
    last_modified = DateTimeField(auto_now=True, editable=False, db_index=True,
                                  verbose_name=_('Updated at'))
    owner = ForeignKey(settings.AUTH_USER_MODEL, verbose_name=_('App owner'),
                       on_delete=CASCADE, related_name='owned_apps')
    co_maintainers = ManyToManyField(settings.AUTH_USER_MODEL, blank=True,
                                     verbose_name=_('Co-Maintainers'),
                                     related_name='co_maintained_apps')
    featured = BooleanField(verbose_name=_('Featured'), default=False)

    class Meta:
        verbose_name = _('App')
        verbose_name_plural = _('Apps')

    def __str__(self) -> str:
        return self.name

    def can_update(self, user: User) -> bool:
        return self.owner == user or user in self.co_maintainers.all()

    def can_delete(self, user: User) -> bool:
        return self.owner == user

    def releases_by_platform_v(self):
        """Looks up all compatible releases for each platform version.
        :return dict with all compatible releases for each platform version.
        """
        releases = {}
        for p_version in settings.PLATFORM_VERSIONS:
            releases[p_version] = self.compatible_releases(p_version)
        return releases

    def latest_releases_by_platform_v(self):
        """Looks up the latest release for each platform version.
        Ignores nightly releases.
        :return dict with the latest release for each platform version.
        """
        all_latest = {}
        for p_version in settings.PLATFORM_VERSIONS:
            compatible = self.compatible_releases(p_version)
            all_latest[p_version] = self._latest_non_nightly(compatible)
        return all_latest

    def compatible_releases(self, platform_version, inclusive=True):
        all_releases = self.releases.all()
        return sorted(list(filter(
            lambda rel: rel.is_compatible(platform_version, inclusive),
            all_releases)), key=lambda rel: Version(rel.version), reverse=True)

    def _latest_non_nightly(self, releases):
        releases = filter(lambda r: not r.version.endswith('-nightly'),
                          releases)
        try:
            return max(releases, key=lambda r: Version(r.version))
        except ValueError:
            return None

class AppRelease(Model):
    version = CharField(max_length=128, verbose_name=_('Version'),
                        help_text=_('Version follows Semantic Versioning'))
    app = ForeignKey('App', on_delete=CASCADE, verbose_name=_('App'),
                     related_name='releases')
    # dependencies
    php_extensions = ManyToManyField('PhpExtension', blank=True,
                                     through='PhpExtensionDependency',
                                     verbose_name=_(
                                         'PHP extension dependency'))
    databases = ManyToManyField('Database', blank=True,
                                through='DatabaseDependency',
                                verbose_name=_('Database dependency'))
    licenses = ManyToManyField('License', verbose_name=_('License'))
    shell_commands = ManyToManyField('ShellCommand', blank=True,
                                     verbose_name=_(
                                         'Shell command dependency'))
    php_version_spec = CharField(max_length=128,
                                 verbose_name=_('PHP version requirement'))
    platform_version_spec = CharField(max_length=128, verbose_name=_(
        'Platform version requirement'))
    min_int_size = IntegerField(blank=True, default=32,
                                verbose_name=_('Minimum Integer Bits'),
                                help_text=_('e.g. 32 for 32bit Integers'))
    checksum = CharField(max_length=64, verbose_name=_('SHA256 checksum'))
    download = URLField(max_length=256, blank=True,
                        verbose_name=_('Archive download Url'))
    created = DateTimeField(auto_now_add=True, editable=False,
                            verbose_name=_('Created at'))
    last_modified = DateTimeField(auto_now=True, editable=False, db_index=True,
                                  verbose_name=_('Updated at'))

    class Meta:
        verbose_name = _('App Release')
        verbose_name_plural = _('App Releases')
        unique_together = (('app', 'version'),)
        ordering = ['-version']

    def can_update(self, user: User) -> bool:
        return self.app.owner == user or user in self.app.co_maintainers.all()

    def can_delete(self, user: User) -> bool:
        return self.can_update(user)

    def __str__(self) -> str:
        return '%s %s' % (self.app, self.version)

    def is_compatible(self, platform_version, inclusive=False):
        """
        Checks if a release is compatible with a platform version
        :param platform_version: the platform version, not required to be
        semver compatible
        :param inclusive: if True the check will also return True if an app
         requires 9.0.1 and the given platform version is 9.0
        :return: True if compatible, otherwise false
        """
        min_version = Version(pad_min_version(platform_version))
        spec = Spec(self.platform_version_spec)
        if inclusive:
            max_version = Version(pad_max_inc_version(platform_version))
            return (min_version in spec or max_version in spec)
        else:
            return min_version in spec

class Screenshot(Model):
    url = URLField(max_length=256, verbose_name=_('Image url'))
    app = ForeignKey('App', on_delete=CASCADE, verbose_name=_('App'),
                     related_name='screenshots')
    ordering = IntegerField(verbose_name=_('Ordering'))

    class Meta:
        verbose_name = _('Screenshot')
        verbose_name_plural = _('Screenshots')
        ordering = ['ordering']

    def __str__(self) -> str:
        return self.url

class ShellCommand(Model):
    name = CharField(max_length=128, unique=True, primary_key=True,
                     verbose_name=_('Shell Command'),
                     help_text=_(
                         'Name of a required shell command, e.g. grep'))

    class Meta:
        verbose_name = _('Shell Command')
        verbose_name_plural = _('Shell Commands')

    def __str__(self) -> str:
        return self.name

class Category(TranslatableModel):
    id = CharField(max_length=128, unique=True, primary_key=True,
                   verbose_name=_('Id'),
                   help_text=_(
                       'Category id which is used to identify a '
                       'category. Used to identify categories when '
                       'uploading an app'))
    created = DateTimeField(auto_now_add=True, editable=False,
                            verbose_name=_('Created at'))
    last_modified = DateTimeField(auto_now=True, editable=False, db_index=True,
                                  verbose_name=_('Updated at'))
    translations = TranslatedFields(
        name=CharField(max_length=128, help_text=_(
            'Category name which will be presented to the user'),
                       verbose_name=_('Name')),
        description=TextField(verbose_name=_('Description'),
                              help_text=_('Will be rendered as Markdown'))
    )

    class Meta:
        verbose_name = _('Category')
        verbose_name_plural = _('Categories')
        ordering = ['id']

    def __str__(self) -> str:
        return self.name

class License(Model):
    id = CharField(max_length=128, unique=True, primary_key=True,
                   verbose_name=_('Id'),
                   help_text=_(
                       'Key which is used to identify a license'))
    name = CharField(max_length=128, verbose_name=_('Name'),
                     help_text=_(
                         'License name which will be presented to '
                         'the user'))

    class Meta:
        verbose_name = _('License')
        verbose_name_plural = _('Licenses')

    def __str__(self) -> str:
        return self.name

class Database(Model):
    id = CharField(max_length=128, unique=True, primary_key=True,
                   verbose_name=_('Id'),
                   help_text=_('Key which is used to identify a database'))
    name = CharField(max_length=128, verbose_name=_('Name'),
                     help_text=_(
                         'Database name which will be presented to the user'))

    class Meta:
        verbose_name = _('Database')
        verbose_name_plural = _('Databases')

    def __str__(self) -> str:
        return self.name

class DatabaseDependency(Model):
    app_release = ForeignKey('AppRelease', on_delete=CASCADE,
                             verbose_name=_('App release'),
                             related_name='databasedependencies')
    database = ForeignKey('Database', related_name='releasedependencies',
                          on_delete=CASCADE, verbose_name=_('Database'))
    version_spec = CharField(max_length=128,
                             verbose_name=_('Database version requirement'))

    class Meta:
        verbose_name = _('Database Dependency')
        verbose_name_plural = _('Database Dependencies')
        unique_together = (('app_release', 'database', 'version_spec'),)

    def __str__(self) -> str:
        return '%s: %s %s' % (self.app_release, self.database,
                              self.version_spec)

class PhpExtension(Model):
    id = CharField(max_length=128, unique=True, help_text=_('e.g. libxml'),
                   primary_key=True, verbose_name=_('PHP extension'))

    class Meta:
        verbose_name = _('PHP Extension')
        verbose_name_plural = _('PHP Extensions')

    def __str__(self) -> str:
        return self.id

class PhpExtensionDependency(Model):
    app_release = ForeignKey('AppRelease', on_delete=CASCADE,
                             verbose_name=_('App Release'),
                             related_name='phpextensiondependencies')
    php_extension = ForeignKey('PhpExtension', on_delete=CASCADE,
                               verbose_name=_('PHP Extension'),
                               related_name='releasedependencies')
    version_spec = CharField(max_length=128,
                             verbose_name=_('Extension version requirement'))

    class Meta:
        verbose_name = _('PHP Extension Dependency')
        verbose_name_plural = _('PHP Extension Dependencies')
        unique_together = (('app_release', 'php_extension', 'version_spec'),)

    def __str__(self) -> str:
        return '%s: %s %s' % (self.app_release.app, self.php_extension,
                              self.version_spec)
<commit_msg>add orderings<commit_after>from functools import reduce
from django.conf import settings  # type: ignore
from django.contrib.auth.models import User  # type: ignore
from django.db.models import ManyToManyField, ForeignKey, \
    URLField, IntegerField, CharField, CASCADE, TextField, \
    DateTimeField, Model, BooleanField, Q  # type: ignore
from django.utils.translation import ugettext_lazy as _  # type: ignore
from nextcloudappstore.core.versioning import pad_min_version, \
    pad_max_inc_version
from parler.models import TranslatedFields, TranslatableModel, \
    TranslatableManager  # type: ignore
from semantic_version import Version, Spec

class AppManager(TranslatableManager):
    def search(self, terms, lang):
        queryset = self.get_queryset().active_translations(lang).language(
            lang).distinct()
        predicates = map(lambda t: (Q(translations__name__icontains=t) |
                                    Q(translations__summary__icontains=t) |
                                    Q(translations__description__icontains=t)),
                         terms)
        query = reduce(lambda x, y: x & y, predicates, Q())
        return queryset.filter(query)

    def get_compatible(self, platform_version, inclusive=False):
        apps = App.objects.prefetch_related('translations', 'screenshots',
                                            'releases', 'releases__databases',
                                            'releases__php_extensions').all()

        def app_filter(app):
            for release in app.releases.all():
                if release.is_compatible(platform_version, inclusive):
                    return True
            return False

        return list(filter(app_filter, apps))

class App(TranslatableModel):
    objects = AppManager()
    id = CharField(max_length=128, unique=True, primary_key=True,
                   verbose_name=_('Id'),
                   help_text=_('app id, identical to folder name'))
    categories = ManyToManyField('Category', verbose_name=_('Category'))
    translations = TranslatedFields(
        name=CharField(max_length=128, verbose_name=_('Name'),
                       help_text=_('Rendered app name for users')),
        summary=CharField(max_length=128, verbose_name=_('Summary'),
                          help_text=_(
                              'Short text describing the app\'s purpose')),
        description=TextField(verbose_name=_('Description'), help_text=_(
            'Will be rendered as Markdown'))
    )
    # resources
    user_docs = URLField(max_length=256, blank=True,
                         verbose_name=_('User documentation url'))
    admin_docs = URLField(max_length=256, blank=True,
                          verbose_name=_('Admin documentation url'))
    developer_docs = URLField(max_length=256, blank=True,
                              verbose_name=_('Developer documentation url'))
    issue_tracker = URLField(max_length=256, blank=True,
                             verbose_name=_('Issue tracker url'))
    website = URLField(max_length=256, blank=True, verbose_name=_('Homepage'))
    created = DateTimeField(auto_now_add=True, editable=False,
                            verbose_name=_('Created at'))
    last_modified = DateTimeField(auto_now=True, editable=False, db_index=True,
                                  verbose_name=_('Updated at'))
    owner = ForeignKey(settings.AUTH_USER_MODEL, verbose_name=_('App owner'),
                       on_delete=CASCADE, related_name='owned_apps')
    co_maintainers = ManyToManyField(settings.AUTH_USER_MODEL, blank=True,
                                     verbose_name=_('Co-Maintainers'),
                                     related_name='co_maintained_apps')
    featured = BooleanField(verbose_name=_('Featured'), default=False)

    class Meta:
        verbose_name = _('App')
        verbose_name_plural = _('Apps')

    def __str__(self) -> str:
        return self.name

    def can_update(self, user: User) -> bool:
        return self.owner == user or user in self.co_maintainers.all()

    def can_delete(self, user: User) -> bool:
        return self.owner == user

    def releases_by_platform_v(self):
        """Looks up all compatible releases for each platform version.
        :return dict with all compatible releases for each platform version.
        """
        releases = {}
        for p_version in settings.PLATFORM_VERSIONS:
            releases[p_version] = self.compatible_releases(p_version)
        return releases

    def latest_releases_by_platform_v(self):
        """Looks up the latest release for each platform version.
        Ignores nightly releases.
        :return dict with the latest release for each platform version.
        """
        all_latest = {}
        for p_version in settings.PLATFORM_VERSIONS:
            compatible = self.compatible_releases(p_version)
            all_latest[p_version] = self._latest_non_nightly(compatible)
        return all_latest

    def compatible_releases(self, platform_version, inclusive=True):
        all_releases = self.releases.all()
        return sorted(list(filter(
            lambda rel: rel.is_compatible(platform_version, inclusive),
            all_releases)), key=lambda rel: Version(rel.version), reverse=True)

    def _latest_non_nightly(self, releases):
        releases = filter(lambda r: not r.version.endswith('-nightly'),
                          releases)
        try:
            return max(releases, key=lambda r: Version(r.version))
        except ValueError:
            return None

class AppRelease(Model):
    version = CharField(max_length=128, verbose_name=_('Version'),
                        help_text=_('Version follows Semantic Versioning'))
    app = ForeignKey('App', on_delete=CASCADE, verbose_name=_('App'),
                     related_name='releases')
    # dependencies
    php_extensions = ManyToManyField('PhpExtension', blank=True,
                                     through='PhpExtensionDependency',
                                     verbose_name=_(
                                         'PHP extension dependency'))
    databases = ManyToManyField('Database', blank=True,
                                through='DatabaseDependency',
                                verbose_name=_('Database dependency'))
    licenses = ManyToManyField('License', verbose_name=_('License'))
    shell_commands = ManyToManyField('ShellCommand', blank=True,
                                     verbose_name=_(
                                         'Shell command dependency'))
    php_version_spec = CharField(max_length=128,
                                 verbose_name=_('PHP version requirement'))
    platform_version_spec = CharField(max_length=128, verbose_name=_(
        'Platform version requirement'))
    min_int_size = IntegerField(blank=True, default=32,
                                verbose_name=_('Minimum Integer Bits'),
                                help_text=_('e.g. 32 for 32bit Integers'))
    checksum = CharField(max_length=64, verbose_name=_('SHA256 checksum'))
    download = URLField(max_length=256, blank=True,
                        verbose_name=_('Archive download Url'))
    created = DateTimeField(auto_now_add=True, editable=False,
                            verbose_name=_('Created at'))
    last_modified = DateTimeField(auto_now=True, editable=False, db_index=True,
                                  verbose_name=_('Updated at'))

    class Meta:
        verbose_name = _('App Release')
        verbose_name_plural = _('App Releases')
        unique_together = (('app', 'version'),)
        ordering = ['-version']

    def can_update(self, user: User) -> bool:
        return self.app.owner == user or user in self.app.co_maintainers.all()

    def can_delete(self, user: User) -> bool:
        return self.can_update(user)

    def __str__(self) -> str:
        return '%s %s' % (self.app, self.version)

    def is_compatible(self, platform_version, inclusive=False):
        """
        Checks if a release is compatible with a platform version
        :param platform_version: the platform version, not required to be
        semver compatible
        :param inclusive: if True the check will also return True if an app
         requires 9.0.1 and the given platform version is 9.0
        :return: True if compatible, otherwise false
        """
        min_version = Version(pad_min_version(platform_version))
        spec = Spec(self.platform_version_spec)
        if inclusive:
            max_version = Version(pad_max_inc_version(platform_version))
            return (min_version in spec or max_version in spec)
        else:
            return min_version in spec

class Screenshot(Model):
    url = URLField(max_length=256, verbose_name=_('Image url'))
    app = ForeignKey('App', on_delete=CASCADE, verbose_name=_('App'),
                     related_name='screenshots')
    ordering = IntegerField(verbose_name=_('Ordering'))

    class Meta:
        verbose_name = _('Screenshot')
        verbose_name_plural = _('Screenshots')
        ordering = ['ordering']

    def __str__(self) -> str:
        return self.url

class ShellCommand(Model):
    name = CharField(max_length=128, unique=True, primary_key=True,
                     verbose_name=_('Shell Command'),
                     help_text=_(
                         'Name of a required shell command, e.g. grep'))

    class Meta:
        verbose_name = _('Shell Command')
        verbose_name_plural = _('Shell Commands')
        ordering = ['name']

    def __str__(self) -> str:
        return self.name

class Category(TranslatableModel):
    id = CharField(max_length=128, unique=True, primary_key=True,
                   verbose_name=_('Id'),
                   help_text=_(
                       'Category id which is used to identify a '
                       'category. Used to identify categories when '
                       'uploading an app'))
    created = DateTimeField(auto_now_add=True, editable=False,
                            verbose_name=_('Created at'))
    last_modified = DateTimeField(auto_now=True, editable=False, db_index=True,
                                  verbose_name=_('Updated at'))
    translations = TranslatedFields(
        name=CharField(max_length=128, help_text=_(
            'Category name which will be presented to the user'),
                       verbose_name=_('Name')),
        description=TextField(verbose_name=_('Description'),
                              help_text=_('Will be rendered as Markdown'))
    )

    class Meta:
        verbose_name = _('Category')
        verbose_name_plural = _('Categories')
        ordering = ['id']

    def __str__(self) -> str:
        return self.name

class License(Model):
    id = CharField(max_length=128, unique=True, primary_key=True,
                   verbose_name=_('Id'),
                   help_text=_(
                       'Key which is used to identify a license'))
    name = CharField(max_length=128, verbose_name=_('Name'),
                     help_text=_(
                         'License name which will be presented to '
                         'the user'))

    class Meta:
        verbose_name = _('License')
        verbose_name_plural = _('Licenses')

    def __str__(self) -> str:
        return self.name

class Database(Model):
    id = CharField(max_length=128, unique=True, primary_key=True,
                   verbose_name=_('Id'),
                   help_text=_('Key which is used to identify a database'))
    name = CharField(max_length=128, verbose_name=_('Name'),
                     help_text=_(
                         'Database name which will be presented to the user'))

    class Meta:
        verbose_name = _('Database')
        verbose_name_plural = _('Databases')
        ordering = ['name']

    def __str__(self) -> str:
        return self.name

class DatabaseDependency(Model):
    app_release = ForeignKey('AppRelease', on_delete=CASCADE,
                             verbose_name=_('App release'),
                             related_name='databasedependencies')
    database = ForeignKey('Database', related_name='releasedependencies',
                          on_delete=CASCADE, verbose_name=_('Database'))
    version_spec = CharField(max_length=128,
                             verbose_name=_('Database version requirement'))

    class Meta:
        verbose_name = _('Database Dependency')
        verbose_name_plural = _('Database Dependencies')
        unique_together = (('app_release', 'database', 'version_spec'),)

    def __str__(self) -> str:
        return '%s: %s %s' % (self.app_release, self.database,
                              self.version_spec)

class PhpExtension(Model):
    id = CharField(max_length=128, unique=True, help_text=_('e.g. libxml'),
                   primary_key=True, verbose_name=_('PHP extension'))

    class Meta:
        verbose_name = _('PHP Extension')
        verbose_name_plural = _('PHP Extensions')
        ordering = ['id']

    def __str__(self) -> str:
        return self.id

class PhpExtensionDependency(Model):
    app_release = ForeignKey('AppRelease', on_delete=CASCADE,
                             verbose_name=_('App Release'),
                             related_name='phpextensiondependencies')
    php_extension = ForeignKey('PhpExtension', on_delete=CASCADE,
                               verbose_name=_('PHP Extension'),
                               related_name='releasedependencies')
    version_spec = CharField(max_length=128,
                             verbose_name=_('Extension version requirement'))

    class Meta:
        verbose_name = _('PHP Extension Dependency')
        verbose_name_plural = _('PHP Extension Dependencies')
        unique_together = (('app_release', 'php_extension', 'version_spec'),)

    def __str__(self) -> str:
        return '%s: %s %s' % (self.app_release.app, self.php_extension,
                              self.version_spec)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright © 2012-2015 Roberto Alsina and others.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from __future__ import print_function
import io
import os
from datetime import datetime
from dateutil.tz import gettz, tzlocal

from nikola.plugin_categories import Command

class CommandDeploy(Command):
    """ Site status. """
    name = "status"

    doc_purpose = "display site status"
    doc_description = "Show information about the posts and site deployment."
    logger = None
    cmd_options = [
        {
            'name': 'list_drafts',
            'short': 'd',
            'long': 'list-drafts',
            'type': bool,
            'default': False,
            'help': 'List all drafts',
        },
        {
            'name': 'list_scheduled',
            'short': 's',
            'long': 'list-scheduled',
            'type': bool,
            'default': False,
            'help': 'List all scheduled posts',
        },
    ]

    def _execute(self, options, args):

        self.site.scan_posts()

        timestamp_path = os.path.join(self.site.config["CACHE_FOLDER"], "lastdeploy")

        last_deploy = None

        try:
            with io.open(timestamp_path, "r", encoding="utf8") as inf:
                last_deploy = datetime.strptime(inf.read().strip(), "%Y-%m-%dT%H:%M:%S.%f")
                last_deploy_offset = datetime.utcnow() - last_deploy
        except (IOError, Exception):
            print("It does not seem like you’ve ever deployed the site (or cache missing).")

        if last_deploy:

            fmod_since_deployment = 0
            for root, dirs, files in os.walk(self.site.config["OUTPUT_FOLDER"], followlinks=True):
                if not dirs and not files:
                    continue
                for fname in files:
                    fpath = os.path.join(root, fname)
                    fmodtime = datetime.fromtimestamp(os.stat(fpath).st_mtime)
                    if fmodtime.replace(tzinfo=tzlocal()) > last_deploy.replace(tzinfo=gettz("UTC")).astimezone(tz=tzlocal()):
                        fmod_since_deployment = fmod_since_deployment + 1

            if fmod_since_deployment > 0:
                print("{0} output files modified since last deployment {1} ago.".format(str(fmod_since_deployment), self.human_time(last_deploy_offset)))
            else:
                print("Last deployment {0} ago.".format(self.human_time(last_deploy_offset)))

        now = datetime.utcnow().replace(tzinfo=gettz("UTC"))

        posts_count = len(self.site.all_posts)

        # find all drafts
        posts_drafts = [post for post in self.site.all_posts if post.is_draft]
        posts_drafts = sorted(posts_drafts, key=lambda post: post.source_path)

        # find all scheduled posts with offset from now until publishing time
        posts_scheduled = [(post.date - now, post) for post in self.site.all_posts if post.publish_later]
        posts_scheduled = sorted(posts_scheduled, key=lambda offset_post: (offset_post[0],offset_post[1].source_path))

        if len(posts_scheduled) > 0:
            if options['list_scheduled']:
                for offset, post in posts_scheduled:
                    print("Scheduled: '{1}' ({2}; source: {3}) in {0}".format(self.human_time(offset), post.meta('title'), post.permalink(), post.source_path))
            else:
                offset, post = posts_scheduled[0]
                print("{0} to next scheduled post ('{1}'; {2}; source: {3}).".format(self.human_time(offset), post.meta('title'), post.permalink(), post.source_path))
        if options['list_drafts']:
            for post in posts_drafts:
                print("Draft: '{0}' ({1}; source: {2})".format(post.meta('title'), post.permalink(), post.source_path))
        print("{0} posts in total, {1} scheduled, and {2} drafts.".format(posts_count, len(posts_scheduled), len(posts_drafts)))

    def human_time(self, dt):
        days = dt.days
        hours = dt.seconds / 60 // 60
        minutes = dt.seconds / 60 - (hours * 60)
        if days > 0:
            return "{0:.0f} days and {1:.0f} hours".format(days, hours)
        elif hours > 0:
            return "{0:.0f} hours and {1:.0f} minutes".format(hours, minutes)
        elif minutes:
            return "{0:.0f} minutes".format(minutes)
        return False
<commit_msg>flake8.<commit_after># -*- coding: utf-8 -*-

# Copyright © 2012-2015 Roberto Alsina and others.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from __future__ import print_function
import io
import os
from datetime import datetime
from dateutil.tz import gettz, tzlocal

from nikola.plugin_categories import Command

class CommandDeploy(Command):
    """ Site status. """
    name = "status"

    doc_purpose = "display site status"
    doc_description = "Show information about the posts and site deployment."
    logger = None
    cmd_options = [
        {
            'name': 'list_drafts',
            'short': 'd',
            'long': 'list-drafts',
            'type': bool,
            'default': False,
            'help': 'List all drafts',
        },
        {
            'name': 'list_scheduled',
            'short': 's',
            'long': 'list-scheduled',
            'type': bool,
            'default': False,
            'help': 'List all scheduled posts',
        },
    ]

    def _execute(self, options, args):

        self.site.scan_posts()

        timestamp_path = os.path.join(self.site.config["CACHE_FOLDER"], "lastdeploy")

        last_deploy = None

        try:
            with io.open(timestamp_path, "r", encoding="utf8") as inf:
                last_deploy = datetime.strptime(inf.read().strip(), "%Y-%m-%dT%H:%M:%S.%f")
                last_deploy_offset = datetime.utcnow() - last_deploy
        except (IOError, Exception):
            print("It does not seem like you’ve ever deployed the site (or cache missing).")

        if last_deploy:

            fmod_since_deployment = 0
            for root, dirs, files in os.walk(self.site.config["OUTPUT_FOLDER"], followlinks=True):
                if not dirs and not files:
                    continue
                for fname in files:
                    fpath = os.path.join(root, fname)
                    fmodtime = datetime.fromtimestamp(os.stat(fpath).st_mtime)
                    if fmodtime.replace(tzinfo=tzlocal()) > last_deploy.replace(tzinfo=gettz("UTC")).astimezone(tz=tzlocal()):
                        fmod_since_deployment = fmod_since_deployment + 1

            if fmod_since_deployment > 0:
                print("{0} output files modified since last deployment {1} ago.".format(str(fmod_since_deployment), self.human_time(last_deploy_offset)))
            else:
                print("Last deployment {0} ago.".format(self.human_time(last_deploy_offset)))

        now = datetime.utcnow().replace(tzinfo=gettz("UTC"))

        posts_count = len(self.site.all_posts)

        # find all drafts
        posts_drafts = [post for post in self.site.all_posts if post.is_draft]
        posts_drafts = sorted(posts_drafts, key=lambda post: post.source_path)

        # find all scheduled posts with offset from now until publishing time
        posts_scheduled = [(post.date - now, post) for post in self.site.all_posts if post.publish_later]
        posts_scheduled = sorted(posts_scheduled, key=lambda offset_post: (offset_post[0], offset_post[1].source_path))

        if len(posts_scheduled) > 0:
            if options['list_scheduled']:
                for offset, post in posts_scheduled:
                    print("Scheduled: '{1}' ({2}; source: {3}) in {0}".format(self.human_time(offset), post.meta('title'), post.permalink(), post.source_path))
            else:
                offset, post = posts_scheduled[0]
                print("{0} to next scheduled post ('{1}'; {2}; source: {3}).".format(self.human_time(offset), post.meta('title'), post.permalink(), post.source_path))
        if options['list_drafts']:
            for post in posts_drafts:
                print("Draft: '{0}' ({1}; source: {2})".format(post.meta('title'), post.permalink(), post.source_path))
        print("{0} posts in total, {1} scheduled, and {2} drafts.".format(posts_count, len(posts_scheduled), len(posts_drafts)))

    def human_time(self, dt):
        days = dt.days
        hours = dt.seconds / 60 // 60
        minutes = dt.seconds / 60 - (hours * 60)
        if days > 0:
            return "{0:.0f} days and {1:.0f} hours".format(days, hours)
        elif hours > 0:
            return "{0:.0f} hours and {1:.0f} minutes".format(hours, minutes)
        elif minutes:
            return "{0:.0f} minutes".format(minutes)
        return False
<|endoftext|>