<commit_before># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from warnings import warn

from datetime import datetime
from sqlalchemy.orm import validates
from pyfarm.warning import ColumnStateChangeWarning
from pyfarm.utility import randint
from pyfarm.flaskapp import db

class StateValidationMixin(object):
    """
    Mixin that adds a `state` column and uses a class
    level `STATE_ENUM` attribute to assist in validation.
    """
    @validates("state")
    def validate_state(self, key, value):
        if value not in self.STATE_ENUM.values():
            # string which represents what states are valid
            valid_states = ", ".join(
                "%s (%s)" % (value, self.STATE_ENUM.get(value))
                for value in sorted(self.STATE_ENUM.values()))

            msg = "%s is not a valid state, valid states " % value
            msg += "are %s" % valid_states
            raise ValueError(msg)

        return value

class RandIdMixin(object):
    """
    Mixin that provides an `id` column which generates a
    value using :func:`.randint`
    """
    id = db.Column(
        db.BigInteger, primary_key=True, default=randint,
        nullable=False, unique=True)

class StateChangedMixin(object):
    """
    Mixin which adds a static method to be used when the model
    state changes
    """
    @staticmethod
    def stateChangedEvent(target, new_value, old_value, initiator):
        """update the datetime objects depending on the new value"""
        if target.id is None:
            pass

        elif new_value == target.STATE_ENUM.RUNNING:
            target.time_started = datetime.now()

            if hasattr(target, "attempts"):
                target.attempts += 1

        elif new_value in (target.STATE_ENUM.DONE, target.STATE_ENUM.FAILED):
            if target.time_started is None:
                msg = "job %s has not been started yet, state is " % target.id
                msg += "being set to %s" % target.STATE_ENUM.get(new_value)
                warn(msg,  ColumnStateChangeWarning)

            target.time_finished = datetime.now()<commit_msg>removing random id mixin<commit_after># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from warnings import warn

from datetime import datetime
from sqlalchemy.orm import validates
from pyfarm.warning import ColumnStateChangeWarning

class StateValidationMixin(object):
    """
    Mixin that adds a `state` column and uses a class
    level `STATE_ENUM` attribute to assist in validation.
    """
    @validates("state")
    def validate_state(self, key, value):
        if value not in self.STATE_ENUM.values():
            # string which represents what states are valid
            valid_states = ", ".join(
                "%s (%s)" % (value, self.STATE_ENUM.get(value))
                for value in sorted(self.STATE_ENUM.values()))

            msg = "%s is not a valid state, valid states " % value
            msg += "are %s" % valid_states
            raise ValueError(msg)

        return value

class StateChangedMixin(object):
    """
    Mixin which adds a static method to be used when the model
    state changes
    """
    @staticmethod
    def stateChangedEvent(target, new_value, old_value, initiator):
        """update the datetime objects depending on the new value"""
        if target.id is None:
            pass

        elif new_value == target.STATE_ENUM.RUNNING:
            target.time_started = datetime.now()

            if hasattr(target, "attempts"):
                target.attempts += 1

        elif new_value in (target.STATE_ENUM.DONE, target.STATE_ENUM.FAILED):
            if target.time_started is None:
                msg = "job %s has not been started yet, state is " % target.id
                msg += "being set to %s" % target.STATE_ENUM.get(new_value)
                warn(msg,  ColumnStateChangeWarning)

            target.time_finished = datetime.now()<|endoftext|>


<commit_before># You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Challenge 10:
- Create 2 servers, supplying a ssh key to be installed at /root/.ssh/authorized_keys.
- Create a load balancer
- Add the 2 new servers to the LB
- Set up LB monitor and custom error page.
- Create a DNS record based on a FQDN for the LB VIP.
- Write the error page html to a file in cloud files for backup.
"""

import os
import sys
import string
import pyrax
import pyrax.exceptions as exc
import time

print("Using credentials file: ~/.rackspace_cloud_credentials")
cred_file = os.path.expanduser("~/.rackspace_cloud_credentials")
try:
    pyrax.set_credential_file(cred_file)
except exc.AuthenticationFailed:
    print("Did you remember to replace the credential file with your actual" +
          "username and api_key?")

if pyrax.identity.authenticated:
    print("Successfully authenticated.")
else:
    print("Authentication failed. Exiting...")
    sys.exit(1)

# variable to hold the count of how many servers will be built
srv_count = 2

print("You are about to create three 512MB Ubuntu 12.04 LTS Cloud Servers.")
print("Please enter the servers' base name (ie. type: web to create servers" +
      "web1, web2, web3, ...)")
print("Base name:"),
cs_base_name = raw_input()

print("Server names will be:")
for i in range(1, srv_count + 1):
    print(cs_base_name + str(i))

ssh_key_location = raw_input("Enter the location of the ssh key file: ")
if not os.path.exists(ssh_key_location):
    print(ssh_key_location +": No such file or directory. Exiting...")
    sys.exit(1)
else:
    with open(ssh_key_location, 'r') as f:
        ssh_key = f.read()

print("SSH key: " + ssh_key)
<commit_msg>Second commit.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2013 Ted Neykov

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Challenge 10:
- Create 2 servers, supplying a ssh key to be installed at
  /root/.ssh/authorized_keys.
- Create a load balancer
- Add the 2 new servers to the LB
- Set up LB monitor and custom error page.
- Create a DNS record based on a FQDN for the LB VIP.
- Write the error page html to a file in cloud files for backup.
"""

import os
import sys
import string
import pyrax
import pyrax.exceptions as exc
import time

print("Using credentials file: ~/.rackspace_cloud_credentials")
cred_file = os.path.expanduser("~/.rackspace_cloud_credentials")
try:
    pyrax.set_credential_file(cred_file)
except exc.AuthenticationFailed:
    print("Did you remember to replace the credential file with your actual" +
          "username and api_key?")

if pyrax.identity.authenticated:
    print("Successfully authenticated.")
else:
    print("Authentication failed. Exiting...")
    sys.exit(1)

# variable to hold the count of how many servers will be built
srv_count = 2
srv_img = "e4dbdba7-b2a4-4ee5-8e8f-4595b6d694ce" # Ubuntu 12.04 LTS (Precise Pangolin)"
srv_flv = 2

print("You are about to create " + str(srv_count) +
      " 512MB Ubuntu 12.04 LTS Cloud Servers.")
print("Please enter the servers' base name (ie. type: web to create servers" +
      "web1, web2, web3, ...)")
print("Base name:"),
cs_base_name = raw_input()

print("Server names will be:")
for i in range(1, srv_count + 1):
    print(cs_base_name + str(i))

ssh_key_location = raw_input("Enter the location of the ssh key file: ")
if not os.path.exists(ssh_key_location):
    print(ssh_key_location +": No such file or directory. Exiting...")
    sys.exit(1)
else:
    with open(ssh_key_location, 'r') as f:
        ssh_key = f.read()

# Build dicst for injecting ssh key
files = {"/root/.ssh/authorized_keys": ssh_key}

answer = raw_input("Proceed with creating server instances? y/n: ")

if answer == "y":
    cs = pyrax.cloudservers
    server_matrix = []

    for s in range(1, srv_count + 1):
        current_name = cs_base_name + str(s)
        print("Creating server: " + current_name)
        # Create server:
        server = cs.servers.create(current_name, srv_img, srv_flv, files=files)
        # Add server information to matrix
        server_matrix.append([str(server.id), str(server.name),
                              str(server.adminPass), ""])
    print("Servers are building...waiting to obtain IP information.")
    received_ips = False
    count_done = 0
    while not received_ips:
        print("Not all IPs have been assigned. (" + str(count_done) + "/" +
              str(srv_count) + ") Sleeping for 30 seconds.")
        time.sleep(30)
        received_ips = True
        # get server list and populate server_matrix with IPs
        for y in cs.servers.list():
            index = 0
            for s in server_matrix:
                if y.id == server_matrix[index][0]:
                    # print(i.networks
                    for k, v in y.networks.iteritems():
                        if k == "public":
                            if len(v[0]) > 15:
                                # print(v[1])
                                server_matrix[index][3] = str(v[1])
                            else:
                                # print(v[0])
                                server_matrix[index][3] = str(v[0])
                            count_done += 1
                index += 1
        count_done = 0
        for x in server_matrix:
            # print(x)
            if len(x[3]) < 1:
                received_ips = False
            else:
                count_done += 1
    print("")
    print("['                uuid                ', ' name '," +
          " 'root password', 'IP address']")
    for x in server_matrix:
        print(x)
else:
    print("Exiting...")
    sys.exit(1)
<|endoftext|>


<commit_before># utilities for loading and saving data

from scipy.io import *
from numpy import *
import pyspark

def parseVector(line, filter="raw", inds=None):

	vec = [float(x) for x in line.split(' ')]
	ts = array(vec[3:]) # get tseries
	if filter == "dff" : # convert to dff
		meanVal = mean(ts)
		ts = (ts - meanVal) / (meanVal + 0.1)
	if inds is not None :
		if inds == "xyz" :
			return ((int(vec[0]),int(vec[1]),int(vec[2])),ts)
		if inds == "linear" :
			k = int(vec[0]) + int((vec[1] - 1)*1650)
			return (k,ts)
	else :
		return ts

def saveout(data, outputDir, outputFile, outputFormat) :

	if outputFormat == "matlab" :
		dtype = type(data)
		if (dtype == pyspark.rdd.RDD) | (dtype == pyspark.rdd.PipelinedRDD) :
			savemat(outputDir+"/"+outputFile+".mat",mdict={outputFile : data.map(float16).collect()},oned_as='column',do_compression='true')
		else :
			savemat(outputDir+"/"+outputFile+".mat",mdict={outputFile : data},oned_as='column',do_compression='true')
<commit_msg>Testing<commit_after># utilities for loading and saving data

from scipy.io import *
from numpy import *
import pyspark

def parseVector(line, filter="raw", inds=None):

	vec = [float(x) for x in line.split(' ')]
	ts = array(vec[3:]) # get tseries
	return ts

def saveout(data, outputDir, outputFile, outputFormat) :

	if outputFormat == "matlab" :
		dtype = type(data)
		if (dtype == pyspark.rdd.RDD) | (dtype == pyspark.rdd.PipelinedRDD) :
			savemat(outputDir+"/"+outputFile+".mat",mdict={outputFile : data.map(float16).collect()},oned_as='column',do_compression='true')
		else :
			savemat(outputDir+"/"+outputFile+".mat",mdict={outputFile : data},oned_as='column',do_compression='true')
<|endoftext|>


<commit_before>import yaml.nodes

__all__ = [
    'construct_from_mapping',
    'construct_from_sequence',
    'construct_from_string',
    'construct_from_integer',
    'construct_from_any',
    'construct_from_none',
    'construct_from_value',
    'construct_from_args_kwargs',
]

def construct_from_mapping(cls, loader, node):
    mapping = loader.construct_mapping(node, True)
    return cls(**mapping)

def construct_from_sequence(cls, loader, node):
    sequence = loader.construct_sequence(node, True)
    return cls(*sequence)

def construct_from_string(cls, loader, node):
    scalar = str(loader.construct_scalar(node))
    return cls(scalar)

def construct_from_integer(cls, loader, node):
    scalar = int(loader.construct_scalar(node))
    return cls(scalar)

def construct_from_none(cls, loader, node):
    assert loader.construct_yaml_null(node) is None
    return cls()

def construct_from_value(cls, loader, node):
    if isinstance(node, yaml.nodes.ScalarNode):
        tag = loader.resolve(yaml.nodes.ScalarNode, node.value, (True, False))
        subnode = yaml.nodes.ScalarNode(tag, node.value, node.start_mark, node.end_mark, node.style)
        value = loader.construct_object(subnode, True)
    elif isinstance(node, yaml.nodes.SequenceNode):
        value = loader.construct_sequence(node, True)
    elif isinstance(node, yaml.nodes.MappingNode):
        value = loader.construct_mapping(node, True)
    else:
        raise ValueError(node)
    return cls(value)

def construct_from_args_kwargs(cls, loader, node):
    mapping = loader.construct_mapping(node, True)
    args = mapping.pop("=", ())
    return cls(*args, **mapping)

def construct_from_any(cls, loader, node):
    if isinstance(node, yaml.nodes.ScalarNode):
        if loader.construct_yaml_null(node) is None:
            return cls()
        else:
            value = loader.construct_scalar(node)
            return cls(value)
    elif isinstance(node, yaml.nodes.SequenceNode):
        return construct_from_sequence(cls, loader, node)
    elif isinstance(node, yaml.nodes.MappingNode):
        return construct_from_args_kwargs(cls, loader, node)
    else:
        raise ValueError(node)
<commit_msg>scalar value fix<commit_after>import yaml.nodes

__all__ = [
    'construct_from_mapping',
    'construct_from_sequence',
    'construct_from_string',
    'construct_from_integer',
    'construct_from_any',
    'construct_from_none',
    'construct_from_value',
    'construct_from_args_kwargs',
]

def construct_from_mapping(cls, loader, node):
    mapping = loader.construct_mapping(node, True)
    return cls(**mapping)

def construct_from_sequence(cls, loader, node):
    sequence = loader.construct_sequence(node, True)
    return cls(*sequence)

def construct_from_string(cls, loader, node):
    scalar = str(loader.construct_scalar(node))
    return cls(scalar)

def construct_from_integer(cls, loader, node):
    scalar = int(loader.construct_scalar(node))
    return cls(scalar)

def construct_from_scalar(cls, loader, node):
    tag = loader.resolve(yaml.nodes.ScalarNode, node.value, (True, False))
    subnode = yaml.nodes.ScalarNode(tag, node.value, node.start_mark, node.end_mark, node.style)
    value = loader.construct_object(subnode, True)
    return cls(value)

def construct_from_none(cls, loader, node):
    assert loader.construct_yaml_null(node) is None
    return cls()

def construct_from_value(cls, loader, node):
    if isinstance(node, yaml.nodes.ScalarNode):
        tag = loader.resolve(yaml.nodes.ScalarNode, node.value, (True, False))
        subnode = yaml.nodes.ScalarNode(tag, node.value, node.start_mark, node.end_mark, node.style)
        value = loader.construct_object(subnode, True)
    elif isinstance(node, yaml.nodes.SequenceNode):
        value = loader.construct_sequence(node, True)
    elif isinstance(node, yaml.nodes.MappingNode):
        value = loader.construct_mapping(node, True)
    else:
        raise ValueError(node)
    return cls(value)

def construct_from_args_kwargs(cls, loader, node):
    mapping = loader.construct_mapping(node, True)
    args = mapping.pop("=", ())
    return cls(*args, **mapping)

def construct_from_any(cls, loader, node):
    if isinstance(node, yaml.nodes.ScalarNode):
        return construct_from_scalar(cls, loader, node)
    elif isinstance(node, yaml.nodes.SequenceNode):
        return construct_from_sequence(cls, loader, node)
    elif isinstance(node, yaml.nodes.MappingNode):
        return construct_from_args_kwargs(cls, loader, node)
    else:
        raise ValueError(node)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Wed Oct 29 13:30:56 2014

@author: ibackus
"""

import os
import glob
import isaac
import re
import subprocess
import ICgen_utils
import ICglobal_settings
global_settings = ICglobal_settings.global_settings

def PBS_script(workdir, param='snapshot.param', nodes=1, ppn=12, walltime=48, \
jobname='PBS_job', backfill=True, email=None):
    """
    A not very robust function to generate PBS submission scripts for ChaNGa
    jobs on hyak.  Some of the requirements include:

    mpi version of ChaNGa
    gcc_4.4.7-ompi_1.6.5
    PBS (it's on hyak, don't worry)

    By default, any directory with 'lastcheckpoint' in it will be restarted!
    If you don't want to just restart a simulation, delete lastcheckpoint from
    the simulation directory!

    **ARGUMENTS**

    *Required*

    workdir : str
        Directory of the simulation

    *Optional*

    param : str
        Filename of the .param file (not full path)
    nodes : int
        Number of computation nodes to request
    ppn : int
        Number of cores per node
    walltime : float or int
        Walltime to request in hours
    jobname : str
    backfill : bool
        Boolean flag for whether to use the backfill (default is TRUE)
    email : str
        Email address for PBS to send updates to

    **RETURN**

    PBS_script : str
        A string for the PBS script.  Can be saved easily to file
    """

    # Setup filenames
    param_full = os.path.join(workdir, param)
    outfile = os.path.join(workdir, 'changa.out')
    fprefix = os.path.splitext(param)[0]

    # Get changa preset
    preset = global_settings['changa_presets']['mpi']

    # Set up the walltime for PBS
    hours = int(walltime)
    mins = int((walltime*60)%60)
    secs = int((walltime*3600)%60)
    walltime_str = '{0:d}:{1:02d}:{2:02d}'.format(hours,mins,secs)

    # Write the script!

    # Start with the shebang
    script = '#!/bin/bash\n'

    # Some PBS flags
    script += '#PBS -N {0}\n\
#PBS -j oe\n\
#PBS -l nodes={1}:ppn={2},feature={2}core\n\
#PBS -l walltime={3}\n\
#PBS -V\n'.format(jobname, nodes, ppn, walltime_str)

    # Email stuff
    if email is not None:

        script += '#PBS -M {0}\n'.format(email)
        script += '#PBS -m be\n'

    # Choose whether to use the backfill
    if backfill:

        script += '#PBS -q bf\n'

    # Runtime initialization
    script += 'module load gcc_4.4.7-ompi_1.6.5\n\
export MX_RCACHE=0\n\
workdir={0}\n\
cd $workdir\n\
changbin=$(which {1})\n'.format(workdir, preset[2])

    # Now assume that we want to restart if there is a checkpoint
    script += 'if [ -e "lastcheckpoint" ]\n\
then\n\
    echo "lastcheckpoint exists -- restarting simulation..."\n\
    last=`cat lastcheckpoint`\n\
    mpirun --mca mtl mx --mca pml cm $changbin +restart {0}.chk$last +balancer MultistepLB_notopo -wall {1} {2} >> {3} 2>&1\n'.format(fprefix,int(walltime*60), param_full, outfile)
    script += 'else\n\
    echo "lastcheckpoint doesnt exist -- starting new simulation..."\n\
    mpirun --mca mtl mx --mca pml cm $changbin -D 3 +consph +balancer MultistepLB_notopo -wall {0} {1} >& {2}\n\
fi\n'.format(int(walltime*60), param_full, outfile)

    return script

def subber(directories, scriptname, scriptstr=None):
    """
    Submits scriptname, contained in all the directories, to the submission
    queue using qsub.

    If set, scriptstr will be used as the submission script

    **ARGUMENTS**

    directories : str, list, ndarray, tuple
        The directory or directories containing the submission script
    scriptname : str
        Script to submit.  Should be present in all the directories
    scriptstr : str, list, or None
        Default = None (do nothing!)
        Either a string containing the PBS submission script (see PBS_script)
        or a list of such strings.  These will be saved to directories/scriptname
    """

    # Make the directories list iterable
    if isinstance(directories, str):

        directories = [directories]

    # Make the submission scripts an iterable list
    if isinstance(scriptstr, str):

        scriptstr = [scriptstr] * len(directories)

    # Get current working directory
    cwd = os.getcwd()

    # Change directories to full paths
    fullpaths = []
    for directory in directories:

        fullpaths.append(os.path.abspath(directory))

    # Submission command
    command = 'qsub ' + scriptname

    # Submit all the scripts
    for i, fullpath in enumerate(fullpaths):

        os.chdir(fullpath)

        # If submission scripts have been provided as strings, write them out
        if scriptstr is not None:

            f = open(scriptname, 'w')
            f.write(scriptstr[i])
            f.close()
            # Make them executable
            os.system('chmod a+x ' + scriptname)

        # Submit the script to PBS
        p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        # Print output
        for line in iter(p.stdout.readline, ''):

            print line,

        p.wait()

    # Return to current working directory
    os.chdir(cwd)

def make_continue_sub(simdir='.', paramname='snapshot.param', \
newparam='continue.param', nSteps=None, nSteps_extra=None, oldsub='subber.sh', \
newsub='cont_subber.sh'):
    """
    Makes a submission script for continuing a simulation from a previous output.
    Also makes a .param file for the continued simulation.  The simulation
    will be continued in the same directory, with snapshot numbering scheme
    for outputs being the same.

    Parameters for the original simulation cannot be altered (except the number
    of steps you want to continue the simulation by).  PBS runtime parameters
    also cannot be changed (number of nodes, walltime, etc...)

    Any checkpoints will be deleted.

    Requires a submission script be present for the original simulation

    NOTE: if nSteps, nSteps_extra are not set, the total number of steps
    to simulate is not changed.

    **ARGUMENTS**

    simdir : str
        The simulation directory
    paramname : str
        Filename of the .param file for the simulation
    newparam : str
        filename for the .param file for the continued simulation
    nSteps : int
        Total number of steps to simulate
    nSteps_extra : int
        Instead of total number of steps this sets how many more to complete
        OVERIDES nSteps!!!
    oldsub : str
        Filename for the original submission script
    newsub : str
        Filename for the new submission script

    **RETURNS**

    sub_path : str
        Full path to the PBS submission script

    """

    # Lazy man's way of dealing with files in another directory
    cwd = os.getcwd()

    os.chdir(simdir)

    # Load param file
    param = isaac.configparser(paramname, 'param')
    fprefix = param['achOutName']

    # Find all the outputs.  They should be of the format fprefix.000000
    search_exp = '^' + fprefix + '.(?:(?<!\d)\d{6}(?!\d))$'
    flist = []

    for fname in glob.glob(fprefix + '*'):

        if re.search(search_exp, fname) is not None:

            flist.append(fname)

    # Find the number of the last output (the last 6 chars should be an int)
    flist.sort()
    iStartStep = int(flist[-1][-6:])
    param['iStartStep'] = iStartStep
    param['achInFile'] = flist[-1]

    # Set the number of steps to run
    if nSteps_extra is not None:

        param['nSteps'] = iStartStep + nSteps_extra

    elif nSteps is not None:

        param['nSteps'] = nSteps

    # Save new param file
    isaac.configsave(param, newparam, ftype='param')

    # Delete old checkpoints

    for checkpoint in glob.glob(fprefix + '.chk*'):

        print 'removing ' + checkpoint
        os.system('rm -rf ' + checkpoint)

    if os.path.exists('lastcheckpoint'):

        print 'removing lastcheckpoint'
        os.remove('lastcheckpoint')

    # Create a submission script for the simulation continuation
    oldsubfile = open(oldsub, 'r')
    newsubfile = open(newsub, 'w')

    for line in oldsubfile:

        newsubfile.write(line.replace(paramname, newparam))

    oldsubfile.close()
    newsubfile.close()

    # Make the submission script executable
    os.chmod(newsub, 0777)

    sub_path = os.path.abspath(newsub)

    # Change back to original working directory
    os.chdir(cwd)

    return sub_path<commit_msg>Updated some comments<commit_after># -*- coding: utf-8 -*-
"""
Contains various functions for submitting ChaNGa jobs to hyak.  These are quite
platform specific and should not be expected to be broadly useful.

Created on Wed Oct 29 13:30:56 2014

@author: ibackus
"""

import os
import glob
import isaac
import re
import subprocess
import ICglobal_settings
global_settings = ICglobal_settings.global_settings

def PBS_script(workdir, param='snapshot.param', nodes=1, ppn=12, walltime=48, \
jobname='PBS_job', backfill=True, email=None):
    """
    A not very robust function to generate PBS submission scripts for ChaNGa
    jobs on hyak.  Some of the requirements include:

    mpi version of ChaNGa
    gcc_4.4.7-ompi_1.6.5
    PBS (it's on hyak, don't worry)

    By default, any directory with 'lastcheckpoint' in it will be restarted!
    If you don't want to just restart a simulation, delete lastcheckpoint from
    the simulation directory!

    **ARGUMENTS**

    *Required*

    workdir : str
        Directory of the simulation

    *Optional*

    param : str
        Filename of the .param file (not full path)
    nodes : int
        Number of computation nodes to request
    ppn : int
        Number of cores per node
    walltime : float or int
        Walltime to request in hours
    jobname : str
    backfill : bool
        Boolean flag for whether to use the backfill (default is TRUE)
    email : str
        Email address for PBS to send updates to

    **RETURN**

    PBS_script : str
        A string for the PBS script.  Can be saved easily to file
    """

    # Setup filenames
    param_full = os.path.join(workdir, param)
    outfile = os.path.join(workdir, 'changa.out')
    fprefix = os.path.splitext(param)[0]

    # Get changa preset
    preset = global_settings['changa_presets']['mpi']

    # Set up the walltime for PBS
    hours = int(walltime)
    mins = int((walltime*60)%60)
    secs = int((walltime*3600)%60)
    walltime_str = '{0:d}:{1:02d}:{2:02d}'.format(hours,mins,secs)

    # Write the script!

    # Start with the shebang
    script = '#!/bin/bash\n'

    # Some PBS flags
    script += '#PBS -N {0}\n\
#PBS -j oe\n\
#PBS -l nodes={1}:ppn={2},feature={2}core\n\
#PBS -l walltime={3}\n\
#PBS -V\n'.format(jobname, nodes, ppn, walltime_str)

    # Email stuff
    if email is not None:

        script += '#PBS -M {0}\n'.format(email)
        script += '#PBS -m be\n'

    # Choose whether to use the backfill
    if backfill:

        script += '#PBS -q bf\n'

    # Runtime initialization
    script += 'module load gcc_4.4.7-ompi_1.6.5\n\
export MX_RCACHE=0\n\
workdir={0}\n\
cd $workdir\n\
changbin=$(which {1})\n'.format(workdir, preset[2])

    # Now assume that we want to restart if there is a checkpoint
    script += 'if [ -e "lastcheckpoint" ]\n\
then\n\
    echo "lastcheckpoint exists -- restarting simulation..."\n\
    last=`cat lastcheckpoint`\n\
    mpirun --mca mtl mx --mca pml cm $changbin +restart {0}.chk$last +balancer MultistepLB_notopo -wall {1} {2} >> {3} 2>&1\n'.format(fprefix,int(walltime*60), param_full, outfile)
    script += 'else\n\
    echo "lastcheckpoint doesnt exist -- starting new simulation..."\n\
    mpirun --mca mtl mx --mca pml cm $changbin -D 3 +consph +balancer MultistepLB_notopo -wall {0} {1} >& {2}\n\
fi\n'.format(int(walltime*60), param_full, outfile)

    return script

def subber(directories, scriptname, scriptstr=None):
    """
    Submits scriptname, contained in all the directories, to the submission
    queue using qsub.

    If set, scriptstr will be used as the submission script

    **ARGUMENTS**

    directories : str, list, ndarray, tuple
        The directory or directories containing the submission script
    scriptname : str
        Script to submit.  Should be present in all the directories
    scriptstr : str, list, or None
        Default = None (do nothing!)
        Either a string containing the PBS submission script (see PBS_script)
        or a list of such strings.  These will be saved to directories/scriptname
    """

    # Make the directories list iterable
    if isinstance(directories, str):

        directories = [directories]

    # Make the submission scripts an iterable list
    if isinstance(scriptstr, str):

        scriptstr = [scriptstr] * len(directories)

    # Get current working directory
    cwd = os.getcwd()

    # Change directories to full paths
    fullpaths = []
    for directory in directories:

        fullpaths.append(os.path.abspath(directory))

    # Submission command
    command = 'qsub ' + scriptname

    # Submit all the scripts
    for i, fullpath in enumerate(fullpaths):

        os.chdir(fullpath)

        # If submission scripts have been provided as strings, write them out
        if scriptstr is not None:

            f = open(scriptname, 'w')
            f.write(scriptstr[i])
            f.close()
            # Make them executable
            os.system('chmod a+x ' + scriptname)

        # Submit the script to PBS
        p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        # Print output
        for line in iter(p.stdout.readline, ''):

            print line,

        p.wait()

    # Return to current working directory
    os.chdir(cwd)

def make_continue_sub(simdir='.', paramname='snapshot.param', \
newparam='continue.param', nSteps=None, nSteps_extra=None, oldsub='subber.sh', \
newsub='cont_subber.sh'):
    """
    Makes a submission script for continuing a simulation from a previous output.
    Also makes a .param file for the continued simulation.  The simulation
    will be continued in the same directory, with snapshot numbering scheme
    for outputs being the same.

    Parameters for the original simulation cannot be altered (except the number
    of steps you want to continue the simulation by).  PBS runtime parameters
    also cannot be changed (number of nodes, walltime, etc...)

    Any checkpoints will be deleted.

    Requires a submission script be present for the original simulation

    NOTE: if nSteps, nSteps_extra are not set, the total number of steps
    to simulate is not changed.

    **ARGUMENTS**

    simdir : str
        The simulation directory
    paramname : str
        Filename of the .param file for the simulation
    newparam : str
        filename for the .param file for the continued simulation
    nSteps : int
        Total number of steps to simulate
    nSteps_extra : int
        Instead of total number of steps this sets how many more to complete
        OVERIDES nSteps!!!
    oldsub : str
        Filename for the original submission script
    newsub : str
        Filename for the new submission script

    **RETURNS**

    sub_path : str
        Full path to the PBS submission script

    """

    # Lazy man's way of dealing with files in another directory
    cwd = os.getcwd()

    os.chdir(simdir)

    # Load param file
    param = isaac.configparser(paramname, 'param')
    fprefix = param['achOutName']

    # Find all the outputs.  They should be of the format fprefix.000000
    search_exp = '^' + fprefix + '.(?:(?<!\d)\d{6}(?!\d))$'
    flist = []

    for fname in glob.glob(fprefix + '*'):

        if re.search(search_exp, fname) is not None:

            flist.append(fname)

    # Find the number of the last output (the last 6 chars should be an int)
    flist.sort()
    iStartStep = int(flist[-1][-6:])
    param['iStartStep'] = iStartStep
    param['achInFile'] = flist[-1]

    # Set the number of steps to run
    if nSteps_extra is not None:

        param['nSteps'] = iStartStep + nSteps_extra

    elif nSteps is not None:

        param['nSteps'] = nSteps

    # Save new param file
    isaac.configsave(param, newparam, ftype='param')

    # Delete old checkpoints

    for checkpoint in glob.glob(fprefix + '.chk*'):

        print 'removing ' + checkpoint
        os.system('rm -rf ' + checkpoint)

    if os.path.exists('lastcheckpoint'):

        print 'removing lastcheckpoint'
        os.remove('lastcheckpoint')

    # Create a submission script for the simulation continuation
    oldsubfile = open(oldsub, 'r')
    newsubfile = open(newsub, 'w')

    for line in oldsubfile:

        newsubfile.write(line.replace(paramname, newparam))

    oldsubfile.close()
    newsubfile.close()

    # Make the submission script executable
    os.chmod(newsub, 0777)

    sub_path = os.path.abspath(newsub)

    # Change back to original working directory
    os.chdir(cwd)

    return sub_path<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4 encoding=utf-8

import sys
import traceback
import time

from django.dispatch import Signal

from ..log.mixin import LoggerMixin
from ..backends.base import BackendBase
from ..apps.base import AppBase
from ..conf import settings

class BaseRouter(object, LoggerMixin):
    """
    """

    incoming_phases = ("filter", "parse", "handle", "default", "cleanup")
    outgoing_phases = ("outgoing",)

    pre_start  = Signal(providing_args=["router"])
    post_start = Signal(providing_args=["router"])
    pre_stop   = Signal(providing_args=["router"])
    post_stop  = Signal(providing_args=["router"])
    post_init  = Signal(providing_args=["router"])

    def __init__(self):

        self.apps = []
        self.backends = {}
        self.logger = None

        self.running = False
        """TODO: Docs"""

    def add_app(self, module_name):
        """
        Find the app named *module_name*, instantiate it, and add it to
        the list of apps to be notified of incoming messages. Return the
        app instance.
        """
        if isinstance(module_name, AppBase):
            app = module_name
        else:
            cls = AppBase.find(module_name)
            if cls is None: return None
            app = cls(self)
        self.apps.append(app)
        return app

    def get_app(self, module_name):
        """Get a handle to one of our apps by module name."""
        cls = AppBase.find(module_name)
        if cls is None: return None

        for app in self.apps:
            if type(app) == cls:
                return app

        raise KeyError("The %s app was not found in the router!" % module_name)

    def add_backend(self, name, module_name, config=None):
        """
        Find the backend named *module_name*, instantiate it, and add it
        to the dict of backends to be polled for incoming messages, once
        the router is started. Return the backend instance.
        """
        if isinstance(module_name, BackendBase):
            backend = module_name
        else:
            cls = BackendBase.find(module_name)
            if cls is None:
                raise ValueError('No such backend "%s"' % module_name)
            config = self._clean_backend_config(config or {})
            backend = cls(self, name, **config)
        self.backends[name] = backend
        return backend

    @staticmethod
    def _clean_backend_config(config):
        """
        Return ``config`` (a dict) with the keys downcased. (This is
        intended to make the backend configuration case insensitive.)
        """

        return dict([
            (key.lower(), val)
            for key, val in config.iteritems()
        ])

    def _start_all_backends(self):
        """
        Start all backends registered in this router.
        """
        for backend in self.backends.values():
            backend.start()

    def _stop_all_backends(self):
        """
        Stop all backends registered in this router.
        """
        for backend in self.backends.values():
            backend.stop()

    def _start_all_apps(self):
        """
        Start all apps registered via Router.add_app.
        """

        for app in self.apps:
            try:
                app.start()

            except:
                app.exception()

    def _stop_all_apps(self):
        """
        Stop all apps registered via Router.add_app.
        """

        for app in self.apps:
            try:
                app.stop()

            except:
                app.exception()

    def start(self):
        """
        Start polling the backends registered via Router.add_backend for
        incoming messages, and keep doing so until a KeyboardInterrupt
        or SystemExit is raised, or Router.stop is called.
        """

        # dump some debug info for now
        #self.info("BACKENDS: %r" % (self.backends))
        #self.info("APPS: %r" % (self.apps))

        self.info("Starting %s..." % settings.PROJECT_NAME)
        self.pre_start.send(self)
        self._start_all_backends()
        self._start_all_apps()
        self.running = True
        self.debug("Started")

    def stop(self, graceful=False):
        """
        Stop the router, which unblocks the Router.start method as soon
        as possible. This may leave unprocessed messages in the incoming
        or outgoing queues.

        If the optional *graceful* argument is True, the router does its
        best to avoid discarding any messages, by refusing to accept new
        incoming messages and blocking (by calling Router.join) until
        all currently pending messages are processed.
        """

        self.running = False

        self.debug("Stopping...")
        self._stop_all_backends()
        self._stop_all_apps()
        self.info("Stopped")

    def incoming_message(self, msg):
        """
        Queue or send immediately the incoming message.  Defaults to sending
        the message immediately.
        """
        self.incoming(msg)

    def incoming(self, msg):
        """
        Incoming phases:

        Filter:
          The first phase, before any actual work is done. This is the
          only phase that can entirely abort further processing of the
          incoming message, which it does by returning True.

        Parse:
          Don't do INSERTs or UPDATEs in here!

        Handle:
          Respond to messages here.

        Default:
          Only called if no responses were sent during the Handle phase.

        Cleanup:
          An opportunity to clean up anything started during earlier phases.
        """

        self.info("Incoming (%s): %s" %\
            (msg.connection, msg.text))

        try:
            for phase in self.incoming_phases:
                self.debug("In %s phase" % phase)

                if phase == "default":
                    if msg.handled:
                        self.debug("Skipping phase")
                        break

                for app in self.apps:
                    self.debug("In %s app" % app)
                    handled = False

                    try:
                        func = getattr(app, phase)
                        handled = func(msg)

                    except Exception, err:
                        app.exception()

                    # during the _filter_ phase, an app can return True
                    # to abort ALL further processing of this message
                    if phase == "filter":
                        if handled is True:
                            self.warning("Message filtered")
                            raise(StopIteration)

                    # during the _handle_ phase, apps can return True
                    # to "short-circuit" this phase, preventing any
                    # further apps from receiving the message
                    elif phase == "handle":
                        if handled is True:
                            self.debug("Short-circuited")
                            # mark the message handled to avoid the
                            # default phase firing unnecessarily
                            msg.handled = True
                            break

                    elif phase == "default":
                        # allow default phase of apps to short circuit
                        # for prioritized contextual responses.
                        if handled is True:
                            self.debug("Short-circuited default")
                            break

        except StopIteration:
            pass

    def outgoing(self, msg):
        """
        """

        self.info("Outgoing (%s): %s" %\
            (msg.connection, msg.text))

        for phase in self.outgoing_phases:
            self.debug("Out %s phase" % phase)
            continue_sending = True

            # call outgoing phases in the opposite order of the incoming
            # phases, so the first app called with an  incoming message
            # is the last app called with an outgoing message
            for app in reversed(self.apps):
                self.debug("Out %s app" % app)

                try:
                    func = getattr(app, phase)
                    continue_sending = func(msg)

                except Exception, err:
                    app.exception()

                # during any outgoing phase, an app can return True to
                # abort ALL further processing of this message
                if continue_sending is False:
                    self.warning("Message cancelled")
                    return False
<commit_msg>a class, not an instance, can be passed to add_backend<commit_after>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4 encoding=utf-8

import sys
import traceback
import time

from django.dispatch import Signal

from ..log.mixin import LoggerMixin
from ..backends.base import BackendBase
from ..apps.base import AppBase
from ..conf import settings

class BaseRouter(object, LoggerMixin):
    """
    """

    incoming_phases = ("filter", "parse", "handle", "default", "cleanup")
    outgoing_phases = ("outgoing",)

    pre_start  = Signal(providing_args=["router"])
    post_start = Signal(providing_args=["router"])
    pre_stop   = Signal(providing_args=["router"])
    post_stop  = Signal(providing_args=["router"])
    post_init  = Signal(providing_args=["router"])

    def __init__(self):

        self.apps = []
        self.backends = {}
        self.logger = None

        self.running = False
        """TODO: Docs"""

    def add_app(self, module_name):
        """
        Find the app named *module_name*, instantiate it, and add it to
        the list of apps to be notified of incoming messages. Return the
        app instance.
        """
        if isinstance(module_name, AppBase):
            app = module_name
        else:
            cls = AppBase.find(module_name)
            if cls is None: return None
            app = cls(self)
        self.apps.append(app)
        return app

    def get_app(self, module_name):
        """Get a handle to one of our apps by module name."""
        cls = AppBase.find(module_name)
        if cls is None: return None

        for app in self.apps:
            if type(app) == cls:
                return app

        raise KeyError("The %s app was not found in the router!" % module_name)

    def add_backend(self, name, module_name, config=None):
        """
        Find the backend named *module_name*, instantiate it, and add it
        to the dict of backends to be polled for incoming messages, once
        the router is started. Return the backend instance.
        """
        if issubclass(module_name, BackendBase):
            cls = module_name
        else:
            cls = BackendBase.find(module_name)
            if cls is None:
                raise ValueError('No such backend "%s"' % module_name)
        config = self._clean_backend_config(config or {})
        backend = cls(self, name, **config)
        self.backends[name] = backend
        return backend

    @staticmethod
    def _clean_backend_config(config):
        """
        Return ``config`` (a dict) with the keys downcased. (This is
        intended to make the backend configuration case insensitive.)
        """

        return dict([
            (key.lower(), val)
            for key, val in config.iteritems()
        ])

    def _start_all_backends(self):
        """
        Start all backends registered in this router.
        """
        for backend in self.backends.values():
            backend.start()

    def _stop_all_backends(self):
        """
        Stop all backends registered in this router.
        """
        for backend in self.backends.values():
            backend.stop()

    def _start_all_apps(self):
        """
        Start all apps registered via Router.add_app.
        """

        for app in self.apps:
            try:
                app.start()

            except:
                app.exception()

    def _stop_all_apps(self):
        """
        Stop all apps registered via Router.add_app.
        """

        for app in self.apps:
            try:
                app.stop()

            except:
                app.exception()

    def start(self):
        """
        Start polling the backends registered via Router.add_backend for
        incoming messages, and keep doing so until a KeyboardInterrupt
        or SystemExit is raised, or Router.stop is called.
        """

        # dump some debug info for now
        #self.info("BACKENDS: %r" % (self.backends))
        #self.info("APPS: %r" % (self.apps))

        self.info("Starting %s..." % settings.PROJECT_NAME)
        self.pre_start.send(self)
        self._start_all_backends()
        self._start_all_apps()
        self.running = True
        self.debug("Started")

    def stop(self, graceful=False):
        """
        Stop the router, which unblocks the Router.start method as soon
        as possible. This may leave unprocessed messages in the incoming
        or outgoing queues.

        If the optional *graceful* argument is True, the router does its
        best to avoid discarding any messages, by refusing to accept new
        incoming messages and blocking (by calling Router.join) until
        all currently pending messages are processed.
        """

        self.running = False

        self.debug("Stopping...")
        self._stop_all_backends()
        self._stop_all_apps()
        self.info("Stopped")

    def incoming_message(self, msg):
        """
        Queue or send immediately the incoming message.  Defaults to sending
        the message immediately.
        """
        self.incoming(msg)

    def incoming(self, msg):
        """
        Incoming phases:

        Filter:
          The first phase, before any actual work is done. This is the
          only phase that can entirely abort further processing of the
          incoming message, which it does by returning True.

        Parse:
          Don't do INSERTs or UPDATEs in here!

        Handle:
          Respond to messages here.

        Default:
          Only called if no responses were sent during the Handle phase.

        Cleanup:
          An opportunity to clean up anything started during earlier phases.
        """

        self.info("Incoming (%s): %s" %\
            (msg.connection, msg.text))

        try:
            for phase in self.incoming_phases:
                self.debug("In %s phase" % phase)

                if phase == "default":
                    if msg.handled:
                        self.debug("Skipping phase")
                        break

                for app in self.apps:
                    self.debug("In %s app" % app)
                    handled = False

                    try:
                        func = getattr(app, phase)
                        handled = func(msg)

                    except Exception, err:
                        app.exception()

                    # during the _filter_ phase, an app can return True
                    # to abort ALL further processing of this message
                    if phase == "filter":
                        if handled is True:
                            self.warning("Message filtered")
                            raise(StopIteration)

                    # during the _handle_ phase, apps can return True
                    # to "short-circuit" this phase, preventing any
                    # further apps from receiving the message
                    elif phase == "handle":
                        if handled is True:
                            self.debug("Short-circuited")
                            # mark the message handled to avoid the
                            # default phase firing unnecessarily
                            msg.handled = True
                            break

                    elif phase == "default":
                        # allow default phase of apps to short circuit
                        # for prioritized contextual responses.
                        if handled is True:
                            self.debug("Short-circuited default")
                            break

        except StopIteration:
            pass

    def outgoing(self, msg):
        """
        """

        self.info("Outgoing (%s): %s" %\
            (msg.connection, msg.text))

        for phase in self.outgoing_phases:
            self.debug("Out %s phase" % phase)
            continue_sending = True

            # call outgoing phases in the opposite order of the incoming
            # phases, so the first app called with an  incoming message
            # is the last app called with an outgoing message
            for app in reversed(self.apps):
                self.debug("Out %s app" % app)

                try:
                    func = getattr(app, phase)
                    continue_sending = func(msg)

                except Exception, err:
                    app.exception()

                # during any outgoing phase, an app can return True to
                # abort ALL further processing of this message
                if continue_sending is False:
                    self.warning("Message cancelled")
                    return False
<|endoftext|>


<commit_before>import csv
import dataserv

def list_to_file(targ_list):
    with open('inv_list','a+') as tempfile:
        for i in range(0, len(targ_list)):
            tempfile.write(repr(targ_list[i]))
            tempfile.write(",")
            tempfile.write("\n")

def extend_list(targ_list):
    idinv_list = []
    for i in targ_list:
        idinv_list.extend(i['Id'])

def recurs_iter(target):
    if type(target) is list:
        src = recurs_iter(iter(target))

    return type(src)

def histogram():
    '''
    using bokeh to visualise:
    from bokeh.plotting import figure, output_file, show
    output_file('histogram.html')
    p = figure(title = 'insert title')
    x = datescount.keys()
    y = datescount.values()
    p.line(x,y)
    show(p)
    '''

    dates = eval(open('dates.txt', 'r+').read())
    from collections import Counter
    datescount = Counter(dates)

    return datescount

def sourcelist(cxn):
    testlist = [cxn.dates(), cxn.tags(), cxn.leadsources()]

    return testlist

def padlist(list1, list2):
    if len(list1)>len(list2):
        padding=(len(list1)-len(list2))*[0]
        list2.extend(padding)
    elif len(list2)>len(list1):
        padding=(len(list2-len(list1))*[0])
        list1.extend(padding)
    else:
        print("Didn't work, give me only 2 lists please")

    ziplist = zip(list1, list2)

    return ziplist

def better_datecompare(date1,date2):
    from datetime import datetime, timedelta
    date1=date1.timetuple()
    date2=date2.timetuple()
    date1=datetime(date1.tm_year,date1.tm_mon,date1.tm_mday)
    date2=datetime(date2.tm_year,date2.tm_mon,date2.tm_mday)
    days=date2-date1

    return days

def convert_infusiondate(IS_dateobject):
    from datetime import datetime
    date = IS_dateobject.timetuple()
    date = datetime(date.tm_year, date.tm_mon, date.tm_mday)

    return date

def linecount(filename):
    if type(filename) != str:
        filename = str(filename)

    with open(filename) as file:
        for i, l in enumerate(file):
            pass

        return i + 1

def leadtime_test():
    import dataserv as ds
    limit=dict(limit=1000)
    lts=ds.LeadtimetoSale().leadtime(**limit)
    ds.Process().procarray(lts)
    for item in lts:
        print(item)

def leadtime_test2():
    import dataserv as ds
    id_pages = ds.Query()._getpages('Contact','Id')
    limit = 999
    lts = []
    for page in range(0, id_pages + 1):
        ref_point = dict(page = page, limit = limit)
        lts.extend(ds.LeadtimetoSale().leadtime(**ref_point))
    ds.Process().procarray(lts)
    for item in lts:
        print(item)

def sendto_sqlite(query_array):
    '''Use sqlite3 module to output to local DB. Saves API calls. Using text datatypes
    in tables to avoid type conversion for datetime objects.
    '''
    import sqlite3
    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()
    c.execute('CREATE TABLE contacts (key text, value integer);')
    for item in query_array:
        # insert item into db. think about datatypes here
        # could possibly just write item as one whole string.
        # read it back in as dict later
        c.executemany('insert into contacts values (?,?);', item.iteritems())
    conn.commit()
    conn.close()

def sendto_json(query_array):
    '''Use json to store entire query as json file.'''
    import json
    with open('dataserv.json') as file:
        for item in diclist:
            json.dump(item, file)

def get_csv(filename):
    if type(filename) != str:
        filename = str(filename)
    csvdata = []
    with open(filename, newline = '') as csvfile:
        reader = csv.reader(csvfile, delimiter = ',')
        csvdata.extend([entry for entry in reader])
    return csvdata

def convert_invoice():
    '''Converts currency column in AUD to float.'''
    import locale
    import sqlite3
    locale.setlocale(locale.LC_ALL, '')
    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()
    c.execute('SELECT [Inv Total], rowid from sales;')
    invoices = c.fetchall()
    for row in invoices:
        invoices[invoices.index(row)] = list(row)
    for invoice in invoices:
        invoice[0] = invoice[0].strip('AUD')
        invoice[0] = invoice[0].strip('-AUD')
        invoice[0] = locale.atof(invoice[0])
    for row in invoices:
        invoices[invoices.index(row)] = tuple(row)
    c.executemany('UPDATE sales set [Inv Total]=? where rowid=?;', invoices)
    conn.commit()
    conn.close()
<commit_msg>Write sql query to extract ledtime data.<commit_after>import csv
import dataserv

def list_to_file(targ_list):
    with open('inv_list','a+') as tempfile:
        for i in range(0, len(targ_list)):
            tempfile.write(repr(targ_list[i]))
            tempfile.write(",")
            tempfile.write("\n")

def extend_list(targ_list):
    idinv_list = []
    for i in targ_list:
        idinv_list.extend(i['Id'])

def recurs_iter(target):
    if type(target) is list:
        src = recurs_iter(iter(target))

    return type(src)

def histogram():
    '''
    using bokeh to visualise:
    from bokeh.plotting import figure, output_file, show
    output_file('histogram.html')
    p = figure(title = 'insert title')
    x = datescount.keys()
    y = datescount.values()
    p.line(x,y)
    show(p)
    '''

    dates = eval(open('dates.txt', 'r+').read())
    from collections import Counter
    datescount = Counter(dates)

    return datescount

def sourcelist(cxn):
    testlist = [cxn.dates(), cxn.tags(), cxn.leadsources()]

    return testlist

def padlist(list1, list2):
    if len(list1)>len(list2):
        padding=(len(list1)-len(list2))*[0]
        list2.extend(padding)
    elif len(list2)>len(list1):
        padding=(len(list2-len(list1))*[0])
        list1.extend(padding)
    else:
        print("Didn't work, give me only 2 lists please")

    ziplist = zip(list1, list2)

    return ziplist

def better_datecompare(date1,date2):
    from datetime import datetime, timedelta
    date1=date1.timetuple()
    date2=date2.timetuple()
    date1=datetime(date1.tm_year,date1.tm_mon,date1.tm_mday)
    date2=datetime(date2.tm_year,date2.tm_mon,date2.tm_mday)
    days=date2-date1

    return days

def convert_infusiondate(IS_dateobject):
    from datetime import datetime
    date = IS_dateobject.timetuple()
    date = datetime(date.tm_year, date.tm_mon, date.tm_mday)

    return date

def linecount(filename):
    if type(filename) != str:
        filename = str(filename)

    with open(filename) as file:
        for i, l in enumerate(file):
            pass

        return i + 1

def leadtime_test():
    import dataserv as ds
    limit=dict(limit=1000)
    lts=ds.LeadtimetoSale().leadtime(**limit)
    ds.Process().procarray(lts)
    for item in lts:
        print(item)

def leadtime_test2():
    import dataserv as ds
    id_pages = ds.Query()._getpages('Contact','Id')
    limit = 999
    lts = []
    for page in range(0, id_pages + 1):
        ref_point = dict(page = page, limit = limit)
        lts.extend(ds.LeadtimetoSale().leadtime(**ref_point))
    ds.Process().procarray(lts)
    for item in lts:
        print(item)

def sendto_sqlite(query_array):
    '''Use sqlite3 module to output to local DB. Saves API calls. Using text datatypes
    in tables to avoid type conversion for datetime objects.
    '''
    import sqlite3
    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()
    c.execute('CREATE TABLE contacts (key text, value integer);')
    for item in query_array:
        # insert item into db. think about datatypes here
        # could possibly just write item as one whole string.
        # read it back in as dict later
        c.executemany('insert into contacts values (?,?);', item.iteritems())
    conn.commit()
    conn.close()

def sendto_json(query_array):
    '''Use json to store entire query as json file.'''
    import json
    with open('dataserv.json') as file:
        for item in diclist:
            json.dump(item, file)

def get_csv(filename):
    if type(filename) != str:
        filename = str(filename)
    csvdata = []
    with open(filename, newline = '') as csvfile:
        reader = csv.reader(csvfile, delimiter = ',')
        csvdata.extend([entry for entry in reader])
    return csvdata

def convert_invoice():
    '''Converts currency column in AUD to float.'''
    import locale
    import sqlite3
    locale.setlocale(locale.LC_ALL, '')
    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()
    c.execute('SELECT [Inv Total], rowid from sales;')
    invoices = c.fetchall()
    for row in invoices:
        invoices[invoices.index(row)] = list(row)
    for invoice in invoices:
        invoice[0] = invoice[0].strip('AUD')
        invoice[0] = invoice[0].strip('-AUD')
        invoice[0] = locale.atof(invoice[0])
    for row in invoices:
        invoices[invoices.index(row)] = tuple(row)
    c.executemany('UPDATE sales set [Inv Total]=? where rowid=?;', invoices)
    conn.commit()
    conn.close()

leadtime_sqlquery =
'''SELECT sales.ContactId, contacts.[Date Created] as entrydate, sales.[Inv Total], sales.Date as saledate
FROM contacts
INNER JOIN sales
ON contacts.Id=sales.ContactId;
'''
<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4

import os, re, traceback
from rapidsms.webui import settings
from django.template import RequestContext
from django.shortcuts import render_to_response as django_r_to_r
from django.core.paginator import Paginator, EmptyPage, InvalidPage

def render_to_response(req, template_name, dictionary=None, **kwargs):
    """Proxies calls to django.shortcuts.render_to_response, to avoid having
       to include the global variables in every request. This is a giant hack,
       and there's probably a much better solution."""

    rs_dict = {
        "apps":  settings.RAPIDSMS_APPS.values(),
        "debug": settings.DEBUG,
        "javascripts": []
    }

    def __js_dir(fs_path, web_prefix):
        """Adds all of the .js files in a given directory to the javascripts array,
           to be included in the <head>. Also checks for a single file js with the
           same name as the directory. (dir_name/*.js and dir_name.js)"""

        if os.path.exists(fs_path):
            rs_dict["javascripts"].extend([
                "%s/%s" % (web_prefix, fn)
                for fn in os.listdir(fs_path)
                if fn[-3:] == ".js"])

        if os.path.exists("%s.js" % (fs_path)):
            rs_dict["javascripts"].append(
                "%s.js" % (web_prefix))

    # add all of the global javascript files for all running
    # apps. this is super handy for packaging functionality
    # which affects the whole webui without hard-coding it
    for app in rs_dict["apps"]:
        __js_dir(
            "%s/static/javascripts/global" % app["path"],
            "/static/%s/javascripts/global" % app["type"])

    # A NEW KIND OF LUNACY: inspect the stack to find out
    # which rapidsms app this function is being called from
    tb = traceback.extract_stack(limit=2)
    m = re.match(r'^.+/(.+?)/views\.py$', tb[-2][0])
    if m is not None:
        app_type = m.group(1)

        # since we're fetching the app conf, add it to the
        # template dict. it wouldn't be a very good idea to
        # use it, but sometimes, when time is short...
        rs_dict["app_conf"] = settings.RAPIDSMS_APPS[app_type]

        # note which app this func was called from, so the tmpl
        # can mark the tab (or some other type of nav) as active
        rs_dict["active_tab"] = rs_dict["app_conf"]["type"]

        # find all of the javascript assets for
        # this app, and add them to the <head>
        __js_dir(
            "%s/static/javascripts/app" % rs_dict["app_conf"]["path"],
            "/static/%s/javascripts/app" % rs_dict["app_conf"]["type"])

        # check for a view-specific javascript,
        # to add LAST, after the dependencies
        view_name = tb[-2][2]
        __js_dir(
            "%s/static/javascripts/page/%s" % (rs_dict["app_conf"]["path"], view_name),
            "/static/%s/javascripts/page/%s.js" % (rs_dict["app_conf"]["type"], view_name))

        # attempt to import the "__global" function from
        # the views.py that this method was called from
        try:
            mod_str = "%s.views" % rs_dict["app_conf"]["module"]
            module = __import__(mod_str, {}, {}, ["__global"])

        except ImportError:
            pass

        # if the views have a __global function, call it with the
        # request object, and add the output (a dictionary) to the
        # rs_dict. note that the 'dictionary' argument to _this_
        # method is merged AFTER this, overriding the global data.
        # also note that we do this here, rather than in the try
        # block above, to avoid masking exceptions raised within
        if module and hasattr(module, "__global"):
            global_data = module.__global(req)
            rs_dict.update(global_data)

    # allow the dict argument to
    # be omitted without blowing up
    if dictionary is not None:
        rs_dict.update(dictionary)

    # unless a context instance has been provided,
    # default to RequestContext, to get all of
    # the TEMPLATE_CONTEXT_PROCESSORS working
    if "context_instance" not in kwargs:
        kwargs["context_instance"] = RequestContext(req)

    # pass on the combined dicts to the original function
    return django_r_to_r(template_name, rs_dict, **kwargs)

def paginated(req, query_set, per_page=20, prefix="", wrapper=None):

    # since the behavior of this function depends on
    # the GET parameters, if there is more than one
    # paginated set per view, we'll need to prefix
    # the parameters to differentiate them
    prefix = ("%s-" % (prefix)) if prefix else ""

    # the per_page argument to this function provides
    # a default, but can be overridden per-request. no
    # interface for this yet, so it's... an easter egg?
    if (prefix + "per-page") in req.GET:
        try:
            per_page = int(req.GET[prefix+"per-page"])

        # if it was provided, it must be valid. we don't
        # want links containing extra useless junk like
        # invalid GET parameters floating around
        except ValueError:
            raise ValueError("Invalid per-page parameter: %r" %
                (req.GET[prefix + "per-page"]))

    try:
        page = int(req.GET.get(prefix+"page", "1"))
        paginator = Paginator(query_set, per_page)
        objects = paginator.page(page)

    # have no mercy if the page parameter is not valid. there
    # should be no links to an invalid page, so coercing it to
    # assume "page=xyz" means "page=1" would just mask bugs
    except (ValueError, EmptyPage, InvalidPage):
        raise ValueError("Invalid Page: %r" %
            (req.GET[prefix + "page"]))

    # if a wrapper function was provided, call it for each
    # object on the page, and replace the list with the result
    if wrapper is not None:
        objects.raw_object_list = objects.object_list
        objects.object_list = map(wrapper, objects.object_list)

    # attach the prefix (if provided; might be blank) to the
    # objects, where it can be found by the {% paginator %} tag
    objects.prefix = prefix

    return objects

def self_link(req, **kwargs):
    new_kwargs = req.GET.copy()

    # build a new querydict using the GET params from the
    # current request, with those passed to this function
    # overridden. we can't use QueryDict.update here, since
    # it APPENDS, rather than REPLACING keys. i hate CGI :|
    for k, v in kwargs.items():
        new_kwargs[k] = v

    # return the same path that we're currently
    # viewing, with the updated query string
    kwargs_enc = new_kwargs.urlencode()
    return "%s?%s" % (req.path, kwargs_enc)

<commit_msg>Making the active tab highlighting work on windows. (cherry picked from commit d372efa33cd9b234e84230a233579d925bdb5504)<commit_after>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4

import os, re, traceback
from rapidsms.webui import settings
from django.template import RequestContext
from django.shortcuts import render_to_response as django_r_to_r
from django.core.paginator import Paginator, EmptyPage, InvalidPage

def render_to_response(req, template_name, dictionary=None, **kwargs):
    """Proxies calls to django.shortcuts.render_to_response, to avoid having
       to include the global variables in every request. This is a giant hack,
       and there's probably a much better solution."""

    rs_dict = {
        "apps":  settings.RAPIDSMS_APPS.values(),
        "debug": settings.DEBUG,
        "javascripts": []
    }

    def __js_dir(fs_path, web_prefix):
        """Adds all of the .js files in a given directory to the javascripts array,
           to be included in the <head>. Also checks for a single file js with the
           same name as the directory. (dir_name/*.js and dir_name.js)"""

        if os.path.exists(fs_path):
            rs_dict["javascripts"].extend([
                "%s/%s" % (web_prefix, fn)
                for fn in os.listdir(fs_path)
                if fn[-3:] == ".js"])

        if os.path.exists("%s.js" % (fs_path)):
            rs_dict["javascripts"].append(
                "%s.js" % (web_prefix))

    # add all of the global javascript files for all running
    # apps. this is super handy for packaging functionality
    # which affects the whole webui without hard-coding it
    for app in rs_dict["apps"]:
        __js_dir(
            "%s/static/javascripts/global" % app["path"],
            "/static/%s/javascripts/global" % app["type"])

    # A NEW KIND OF LUNACY: inspect the stack to find out
    # which rapidsms app this function is being called from
    tb = traceback.extract_stack(limit=2)
    sep = os.sep
    if sep == '\\':
        # if windows, the file separator itself needs to be
        # escaped again
        sep = "\\\\"
    m = re.match(r'^.+%s(.+?)%sviews\.py$' % (sep, sep), tb[-2][0])
    if m is not None:
        app_type = m.group(1)

        # since we're fetching the app conf, add it to the
        # template dict. it wouldn't be a very good idea to
        # use it, but sometimes, when time is short...
        rs_dict["app_conf"] = settings.RAPIDSMS_APPS[app_type]

        # note which app this func was called from, so the tmpl
        # can mark the tab (or some other type of nav) as active
        rs_dict["active_tab"] = rs_dict["app_conf"]["type"]

        # find all of the javascript assets for
        # this app, and add them to the <head>
        __js_dir(
            "%s/static/javascripts/app" % rs_dict["app_conf"]["path"],
            "/static/%s/javascripts/app" % rs_dict["app_conf"]["type"])

        # check for a view-specific javascript,
        # to add LAST, after the dependencies
        view_name = tb[-2][2]
        __js_dir(
            "%s/static/javascripts/page/%s" % (rs_dict["app_conf"]["path"], view_name),
            "/static/%s/javascripts/page/%s.js" % (rs_dict["app_conf"]["type"], view_name))

        # attempt to import the "__global" function from
        # the views.py that this method was called from
        try:
            mod_str = "%s.views" % rs_dict["app_conf"]["module"]
            module = __import__(mod_str, {}, {}, ["__global"])

        except ImportError:
            pass

        # if the views have a __global function, call it with the
        # request object, and add the output (a dictionary) to the
        # rs_dict. note that the 'dictionary' argument to _this_
        # method is merged AFTER this, overriding the global data.
        # also note that we do this here, rather than in the try
        # block above, to avoid masking exceptions raised within
        if module and hasattr(module, "__global"):
            global_data = module.__global(req)
            rs_dict.update(global_data)

    # allow the dict argument to
    # be omitted without blowing up
    if dictionary is not None:
        rs_dict.update(dictionary)

    # unless a context instance has been provided,
    # default to RequestContext, to get all of
    # the TEMPLATE_CONTEXT_PROCESSORS working
    if "context_instance" not in kwargs:
        kwargs["context_instance"] = RequestContext(req)

    # pass on the combined dicts to the original function
    return django_r_to_r(template_name, rs_dict, **kwargs)

def paginated(req, query_set, per_page=20, prefix="", wrapper=None):

    # since the behavior of this function depends on
    # the GET parameters, if there is more than one
    # paginated set per view, we'll need to prefix
    # the parameters to differentiate them
    prefix = ("%s-" % (prefix)) if prefix else ""

    # the per_page argument to this function provides
    # a default, but can be overridden per-request. no
    # interface for this yet, so it's... an easter egg?
    if (prefix + "per-page") in req.GET:
        try:
            per_page = int(req.GET[prefix+"per-page"])

        # if it was provided, it must be valid. we don't
        # want links containing extra useless junk like
        # invalid GET parameters floating around
        except ValueError:
            raise ValueError("Invalid per-page parameter: %r" %
                (req.GET[prefix + "per-page"]))

    try:
        page = int(req.GET.get(prefix+"page", "1"))
        paginator = Paginator(query_set, per_page)
        objects = paginator.page(page)

    # have no mercy if the page parameter is not valid. there
    # should be no links to an invalid page, so coercing it to
    # assume "page=xyz" means "page=1" would just mask bugs
    except (ValueError, EmptyPage, InvalidPage):
        raise ValueError("Invalid Page: %r" %
            (req.GET[prefix + "page"]))

    # if a wrapper function was provided, call it for each
    # object on the page, and replace the list with the result
    if wrapper is not None:
        objects.raw_object_list = objects.object_list
        objects.object_list = map(wrapper, objects.object_list)

    # attach the prefix (if provided; might be blank) to the
    # objects, where it can be found by the {% paginator %} tag
    objects.prefix = prefix

    return objects

def self_link(req, **kwargs):
    new_kwargs = req.GET.copy()

    # build a new querydict using the GET params from the
    # current request, with those passed to this function
    # overridden. we can't use QueryDict.update here, since
    # it APPENDS, rather than REPLACING keys. i hate CGI :|
    for k, v in kwargs.items():
        new_kwargs[k] = v

    # return the same path that we're currently
    # viewing, with the updated query string
    kwargs_enc = new_kwargs.urlencode()
    return "%s?%s" % (req.path, kwargs_enc)

<|endoftext|>


<commit_before>import csv
import dataserv

def list_to_file(targ_list):
    with open('inv_list','a+') as tempfile:
        for i in range(0, len(targ_list)):
            tempfile.write(repr(targ_list[i]))
            tempfile.write(",")
            tempfile.write("\n")

def extend_list(targ_list):
    idinv_list = []
    for i in targ_list:
        idinv_list.extend(i['Id'])

def recurs_iter(target):
    if type(target) is list:
        src = recurs_iter(iter(target))

    return type(src)

def histogram():
    '''
    using bokeh to visualise:
    from bokeh.plotting import figure, output_file, show
    output_file('histogram.html')
    p = figure(title = 'insert title')
    x = datescount.keys()
    y = datescount.values()
    p.line(x,y)
    show(p)
    '''

    dates = eval(open('dates.txt', 'r+').read())
    from collections import Counter
    datescount = Counter(dates)

    return datescount

def sourcelist(cxn):
    testlist = [cxn.dates(), cxn.tags(), cxn.leadsources()]

    return testlist

def padlist(list1, list2):
    if len(list1)>len(list2):
        padding=(len(list1)-len(list2))*[0]
        list2.extend(padding)
    elif len(list2)>len(list1):
        padding=(len(list2-len(list1))*[0])
        list1.extend(padding)
    else:
        print("Didn't work, give me only 2 lists please")

    ziplist = zip(list1, list2)

    return ziplist

def better_datecompare(date1,date2):
    from datetime import datetime, timedelta
    date1=date1.timetuple()
    date2=date2.timetuple()
    date1=datetime(date1.tm_year,date1.tm_mon,date1.tm_mday)
    date2=datetime(date2.tm_year,date2.tm_mon,date2.tm_mday)
    days=date2-date1

    return days

def convert_infusiondate(IS_dateobject):
    from datetime import datetime
    date = IS_dateobject.timetuple()
    date = datetime(date.tm_year, date.tm_mon, date.tm_mday)

    return date

def linecount(filename):
    if type(filename) != str:
        filename = str(filename)

    with open(filename) as file:
        for i, l in enumerate(file):
            pass

        return i + 1

def leadtime_test():
    import dataserv as ds
    limit=dict(limit=1000)
    lts=ds.LeadtimetoSale().leadtime(**limit)
    ds.Process().procarray(lts)
    for item in lts:
        print(item)

def leadtime_test2():
    import dataserv as ds
    id_pages = ds.Query()._getpages('Contact','Id')
    limit = 999
    lts = []
    for page in range(0, id_pages + 1):
        ref_point = dict(page = page, limit = limit)
        lts.extend(ds.LeadtimetoSale().leadtime(**ref_point))
    ds.Process().procarray(lts)
    for item in lts:
        print(item)

def sendto_sqlite(query_array):
    '''Use sqlite3 module to output to local DB. Saves API calls. Using text datatypes
    in tables to avoid type conversion for datetime objects.
    '''
    import sqlite3
    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()
    c.execute('CREATE TABLE contacts (key text, value integer);')
    for item in query_array:
        # insert item into db. think about datatypes here
        # could possibly just write item as one whole string.
        # read it back in as dict later
        c.executemany('insert into contacts values (?,?);', item.iteritems())
    conn.commit()
    conn.close()

def sendto_json(query_array):
    '''Use json to store entire query as json file.'''
    import json
    with open('dataserv.json') as file:
        for item in diclist:
            json.dump(item, file)

def get_csv(filename):
    if type(filename) != str:
        filename = str(filename)
    csvdata = []
    with open(filename, newline = '') as csvfile:
        reader = csv.reader(csvfile, delimiter = ',')
        csvdata.extend([entry for entry in reader])
    return csvdata

def convert_invoice():
    '''Converts currency column in AUD to float.'''
    import locale
    import sqlite3
    locale.setlocale(locale.LC_ALL, '')
    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()
    c.execute('SELECT [Inv Total], rowid from sales;')
    invoices = c.fetchall()
    for row in invoices:
        invoices[invoices.index(row)] = list(row)
    for invoice in invoices:
        invoice[0] = invoice[0].strip('AUD')
        invoice[0] = invoice[0].strip('-AUD')
        invoice[0] = locale.atof(invoice[0])
    for row in invoices:
        invoices[invoices.index(row)] = tuple(row)
    c.executemany('UPDATE sales set [Inv Total]=? where rowid=?;', invoices)
    conn.commit()
    conn.close()

def create_joinlisttable():
    import sqlite3

    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()

    join_contacts_invoices = '''\
    SELECT contacts.Id, contacts.[Date Created], contacts.[Lead Source],\
    sales.[Inv Total], sales.Date \
    FROM contacts INNER JOIN sales \
    ON contacts.Id = sales.ContactId;\
    '''
    c.execute(join_contacts_invoices)
    joinlist = c.fetchall()
    joinlist.sort(key = lambda x: x[0])

    c.execute('''CREATE TABLE contactsales(
    contactid text, entrydate text, leadsource text, invamount text, invdate text);''')
    c.executemany('INSERT INTO contactsales VALUES (?,?,?,?,?);', joinlist)

    conn.commit()
    conn.close()

def get_invoicedates():
    import sqlite3

    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()
    conn.text_factory = int
    c.execute('SELECT Id FROM contacts;')
    contact_idlist = c.fetchall()
    contact_invlist = dict()
    conn.text_factory = str
    for cid in contact_idlist:
        c.execute('SELECT Date FROM sales where sales.ContactId = (?);', cid)
        contact_invlist[cid] = c.fetchall()
    conn.close()
    return contact_invlist

def leadtime_fromdb(datecreated, invdates):
    '''
    datecreated: contact creation date
    invdates: list of invoice dates for same contact
    '''

    dt_creat = convert_datestring(datecreated)
    new_invdates = [convert_datestring(date) for date in invdates]
    firstsale = min(new_invdates)
    leadtime = firstsale - dt_creat

    return leadtime  # this will be number of days

def convert_datestring(targetdate):
    import time
    seconds_per_day = 60*60*24
    newdate = time.strptime(targetdate.split()[0], '%d/%m/%Y')
    newdate = time.mktime(newdate)
    newdate = newdate // seconds_per_day

    return newdate

def list_convert(targetlist):
    newlist = [list(row) for row in targetlist]
    for newrow in newlist:
        newrow[1] = convert_datestring(newrow[1])
        newrow[4] = convert_datestring(newrow[4])

    return newlist

def get_db_table(db_name, db_table):
    import sqlite3

    conn = sqlite3.connect(db_name)
    c = conn.cursor()
    c.execute('SELECT * FROM {}'.format(db_table))
    db_tbl = c.fetchall()

    return db_tbl

def leadtime_from_db(targetlist):
    newlist = dict()
    for row in targetlist:
        if row[0] not in newlist.keys():
            newlist[row[0]] = dict(entrydate = row[1], invdates = [row[4]])
        else:
            newlist[row[0]]['invdates'].append(row[4])

        leadtime = min(newlist[row[0]]['invdates']) - newlist[row[0]]['entrydate']
        newlist[row[0]]['leadtime'] = leadtime

    return newlist

def get_data():
    data = get_db_table('dataserv.db', 'contactsales')
    data = list_convert(data)
    data = leadtime_from_db(data)

    return data

def get_leadtime():
    leadtime = [row['leadtime'] for row in get_data().values()]

    return leadtime

def stats_leadtime():
    lt = get_leadtime()
    average_leadtime = sum(lt)/len(lt)
    std_dev = statistics.pstdev(lt)
    quintile_5 = 0.8*len(lt)
    eightypercentofsales = lt[quintile_5]
    median_leadtime = statistics.median(lt)

    stats = dict(average_leadtime = average_leadtime,
                standard_deviation = standard_deviation,
                eightypercent = eightypercentofsales,
                median = meadian_leadtime,
                fulllist = lt)

    return stats
<commit_msg>Fix names, float and int types in stats_leadtime().<commit_after>import csv
import dataserv
import statistics

def list_to_file(targ_list):
    with open('inv_list','a+') as tempfile:
        for i in range(0, len(targ_list)):
            tempfile.write(repr(targ_list[i]))
            tempfile.write(",")
            tempfile.write("\n")

def extend_list(targ_list):
    idinv_list = []
    for i in targ_list:
        idinv_list.extend(i['Id'])

def recurs_iter(target):
    if type(target) is list:
        src = recurs_iter(iter(target))

    return type(src)

def histogram():
    '''
    using bokeh to visualise:
    from bokeh.plotting import figure, output_file, show
    output_file('histogram.html')
    p = figure(title = 'insert title')
    x = datescount.keys()
    y = datescount.values()
    p.line(x,y)
    show(p)
    '''

    dates = eval(open('dates.txt', 'r+').read())
    from collections import Counter
    datescount = Counter(dates)

    return datescount

def sourcelist(cxn):
    testlist = [cxn.dates(), cxn.tags(), cxn.leadsources()]

    return testlist

def padlist(list1, list2):
    if len(list1)>len(list2):
        padding=(len(list1)-len(list2))*[0]
        list2.extend(padding)
    elif len(list2)>len(list1):
        padding=(len(list2-len(list1))*[0])
        list1.extend(padding)
    else:
        print("Didn't work, give me only 2 lists please")

    ziplist = zip(list1, list2)

    return ziplist

def better_datecompare(date1,date2):
    from datetime import datetime, timedelta
    date1=date1.timetuple()
    date2=date2.timetuple()
    date1=datetime(date1.tm_year,date1.tm_mon,date1.tm_mday)
    date2=datetime(date2.tm_year,date2.tm_mon,date2.tm_mday)
    days=date2-date1

    return days

def convert_infusiondate(IS_dateobject):
    from datetime import datetime
    date = IS_dateobject.timetuple()
    date = datetime(date.tm_year, date.tm_mon, date.tm_mday)

    return date

def linecount(filename):
    if type(filename) != str:
        filename = str(filename)

    with open(filename) as file:
        for i, l in enumerate(file):
            pass

        return i + 1

def leadtime_test():
    import dataserv as ds
    limit=dict(limit=1000)
    lts=ds.LeadtimetoSale().leadtime(**limit)
    ds.Process().procarray(lts)
    for item in lts:
        print(item)

def leadtime_test2():
    import dataserv as ds
    id_pages = ds.Query()._getpages('Contact','Id')
    limit = 999
    lts = []
    for page in range(0, id_pages + 1):
        ref_point = dict(page = page, limit = limit)
        lts.extend(ds.LeadtimetoSale().leadtime(**ref_point))
    ds.Process().procarray(lts)
    for item in lts:
        print(item)

def sendto_sqlite(query_array):
    '''Use sqlite3 module to output to local DB. Saves API calls. Using text datatypes
    in tables to avoid type conversion for datetime objects.
    '''
    import sqlite3
    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()
    c.execute('CREATE TABLE contacts (key text, value integer);')
    for item in query_array:
        # insert item into db. think about datatypes here
        # could possibly just write item as one whole string.
        # read it back in as dict later
        c.executemany('insert into contacts values (?,?);', item.iteritems())
    conn.commit()
    conn.close()

def sendto_json(query_array):
    '''Use json to store entire query as json file.'''
    import json
    with open('dataserv.json') as file:
        for item in diclist:
            json.dump(item, file)

def get_csv(filename):
    if type(filename) != str:
        filename = str(filename)
    csvdata = []
    with open(filename, newline = '') as csvfile:
        reader = csv.reader(csvfile, delimiter = ',')
        csvdata.extend([entry for entry in reader])
    return csvdata

def convert_invoice():
    '''Converts currency column in AUD to float.'''
    import locale
    import sqlite3
    locale.setlocale(locale.LC_ALL, '')
    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()
    c.execute('SELECT [Inv Total], rowid from sales;')
    invoices = c.fetchall()
    for row in invoices:
        invoices[invoices.index(row)] = list(row)
    for invoice in invoices:
        invoice[0] = invoice[0].strip('AUD')
        invoice[0] = invoice[0].strip('-AUD')
        invoice[0] = locale.atof(invoice[0])
    for row in invoices:
        invoices[invoices.index(row)] = tuple(row)
    c.executemany('UPDATE sales set [Inv Total]=? where rowid=?;', invoices)
    conn.commit()
    conn.close()

def create_joinlisttable():
    import sqlite3

    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()

    join_contacts_invoices = '''\
    SELECT contacts.Id, contacts.[Date Created], contacts.[Lead Source],\
    sales.[Inv Total], sales.Date \
    FROM contacts INNER JOIN sales \
    ON contacts.Id = sales.ContactId;\
    '''
    c.execute(join_contacts_invoices)
    joinlist = c.fetchall()
    joinlist.sort(key = lambda x: x[0])

    c.execute('''CREATE TABLE contactsales(
    contactid text, entrydate text, leadsource text, invamount text, invdate text);''')
    c.executemany('INSERT INTO contactsales VALUES (?,?,?,?,?);', joinlist)

    conn.commit()
    conn.close()

def get_invoicedates():
    import sqlite3

    conn = sqlite3.connect('dataserv.db')
    c = conn.cursor()
    conn.text_factory = int
    c.execute('SELECT Id FROM contacts;')
    contact_idlist = c.fetchall()
    contact_invlist = dict()
    conn.text_factory = str
    for cid in contact_idlist:
        c.execute('SELECT Date FROM sales where sales.ContactId = (?);', cid)
        contact_invlist[cid] = c.fetchall()
    conn.close()
    return contact_invlist

def leadtime_fromdb(datecreated, invdates):
    '''
    datecreated: contact creation date
    invdates: list of invoice dates for same contact
    '''

    dt_creat = convert_datestring(datecreated)
    new_invdates = [convert_datestring(date) for date in invdates]
    firstsale = min(new_invdates)
    leadtime = firstsale - dt_creat

    return leadtime  # this will be number of days

def convert_datestring(targetdate):
    import time
    seconds_per_day = 60*60*24
    newdate = time.strptime(targetdate.split()[0], '%d/%m/%Y')
    newdate = time.mktime(newdate)
    newdate = newdate // seconds_per_day

    return newdate

def list_convert(targetlist):
    newlist = [list(row) for row in targetlist]
    for newrow in newlist:
        newrow[1] = convert_datestring(newrow[1])
        newrow[4] = convert_datestring(newrow[4])

    return newlist

def get_db_table(db_name, db_table):
    import sqlite3

    conn = sqlite3.connect(db_name)
    c = conn.cursor()
    c.execute('SELECT * FROM {}'.format(db_table))
    db_tbl = c.fetchall()

    return db_tbl

def leadtime_from_db(targetlist):
    newlist = dict()
    for row in targetlist:
        if row[0] not in newlist.keys():
            newlist[row[0]] = dict(entrydate = row[1], invdates = [row[4]])
        else:
            newlist[row[0]]['invdates'].append(row[4])

        leadtime = min(newlist[row[0]]['invdates']) - newlist[row[0]]['entrydate']
        newlist[row[0]]['leadtime'] = leadtime

    return newlist

def get_data():
    data = get_db_table('dataserv.db', 'contactsales')
    data = list_convert(data)
    data = leadtime_from_db(data)

    return data

def get_leadtime():
    leadtime = [row['leadtime'] for row in get_data().values()]

    return leadtime

def stats_leadtime():
    lt = get_leadtime()
    average_leadtime = sum(lt)/len(lt)
    std_dev = statistics.pstdev(lt)
    quintile_5 = int(0.8*len(lt))
    eightypercentofsales = lt[quintile_5]
    median_leadtime = statistics.median(lt)

    stats = dict(average_leadtime = average_leadtime,
                standard_deviation = std_dev,
                eightypercent = eightypercentofsales,
                median = median_leadtime,
                fulllist = lt)

    return stats
<|endoftext|>