<commit_before>'''
Created on May 23, 2014

@author: sdc50
'''
#TODO: add ability to get stats about the job (i.e. number of jobs, run time, etc.)
#TODO: add ability to submit to remote schedulers

import os, subprocess, re
from collections import OrderedDict

class Job(object):
    """classdocs

    http://research.cs.wisc.edu/htcondor/manual/v7.8/condor_submit.html#man-condor-submit

    """

    def __init__(self, name, attributes=None, executable=None, arguments=None, num_jobs=None):
        """Constructor

        """
        if attributes:
            assert isinstance(attributes, dict)
        self.__dict__['_name'] = name
        self.__dict__['_attributes'] = attributes or OrderedDict()
        self.__dict__['_num_jobs'] = 1
        self.__dict__['_cluster_id'] = None
        self.__dict__['_job_file'] = ''
        self.job_name = name
        self.executable = executable
        self.arguments = arguments

    def __str__(self):
        """docstring

        """

        return '\n'.join(self._list_attributes()) + '\n\nqueue %d\n' % (self.num_jobs)

    def __repr__(self):
        """docstring

        """
        return self.attributes.__repr__()

    def __getattr__(self, item):
        """

        :param item:
        :return:
        """
        self.get(item)

    def __setattr__(self, key, value):
        """

        :param key:
        :param value:
        :return:
        """
        self.set(key, value)

    @property
    def name(self):
        """

        :return:
        """
        self._name = self.get('job_name')
        return self._name

    @name.setter
    def name(self,name):
        """

        :param name:
        :return:
        """
        self.set('job_name', name)

    @property
    def attributes(self):
        """

        :return:
        """
        return self._attributes

    @property
    def num_jobs(self):
        """

        :return:
        """
        return self._num_jobs

    @num_jobs.setter
    def num_jobs(self, num_jobs):
        """

        :param num_jobs:
        :return:
        """
        self._num_jobs = int(num_jobs)

    @property
    def cluster_id(self):
        """

        :return:
        """
        return self._cluster_id

    @property
    def job_file(self):
        """

        :return:
        """
        #TODO: should the job file be just the name or the name and initdir?
        job_file_name = '%s.job' % (self.name)
        job_file_path = os.path.join(self.initialdir,job_file_name)
        self._job_file = job_file_path
        return self._job_file

    @property
    def log_file(self):
        """

        :return:
        """
        log_file = self.get('log')
        if not log_file:
            log_file = '%s.log' % (self.name)
            self.set('log', log_file)
        return self._resolve_attribute('log')

    @property
    def initialdir(self):
        """

        :return:
        """
        initial_dir = self._resolve_attribute('initialdir')
        if not initial_dir:
            initial_dir = os.getcwd()
        return initial_dir

    def submit(self, queue=None, options=None):
        """docstring

        """

        if not self.executable:
            raise NoExecutable('You cannot submit a job without an executable')

        self._num_jobs = queue or self.num_jobs

        self._write_job_file()

        args = ['condor_submit']
        if options:
            args.append(options)
        args.append(self.job_file)

        process = subprocess.Popen(args, stdout = subprocess.PIPE, stderr=subprocess.PIPE)
        out,err = process.communicate()

        if err:
            if re.match('WARNING',err):
                print(err)
            else:
                raise Exception(err)
        print out
        try:
            self._cluster_id = int(re.search('(?<=cluster |\*\* Proc )(\d*)', out).group(1))
        except:
            self._cluster_id = -1
        return self.cluster_id

    def remove(self):
        """docstring

        """
        options = self.cluster_id ##TODO allow other options (like specific processes in a cluster to be removed)
        args = ['condor_rm', options]
        process = subprocess.Popen(args, stdout = subprocess.PIPE, stderr=subprocess.PIPE)
        out,err = process.communicate()
        print(out,err)

    def edit(self):
        """interface for CLI edit commands

        """
        raise NotImplementedError("This method is not yet implemented")

    def status(self):
        """docstring

        """
        raise NotImplementedError("This method is not yet implemented")

    def wait(self):
        """

        :return:
        """
        process = subprocess.Popen(['condor_wait', self.log_file], stdout = subprocess.PIPE, stderr=subprocess.PIPE)
        process.communicate()

    def get(self, attr, value=None):
        """get attribute from job file

        """
        try:
            value = self.attributes[attr]
        except KeyError:
            pass
        return value

    def set(self, attr, value):
        """set attribute in job file

        """
        self.attributes[attr] = value

    def delete(self, attr):
        """delete attribute from job file
        :param attr:
        :return:none
        """
        self.attributes.pop(attr)

    def _write_job_file(self):
        self._make_job_dirs()
        job_file = open(self.job_file, 'w')
        job_file.write(self.__str__())
        job_file.close()

    def _list_attributes(self):
        list = []
        for k,v in self.attributes.iteritems():
            if v:
                list.append(k + ' = ' + str(v))
        return list

    def _make_dir(self, dir_name):
        """docstring

        """
        try:
            os.makedirs(dir_name)
        except OSError:
            pass

    def _make_job_dirs(self):
        """docstring

        """
        self._make_dir(self.initialdir)
        log_dir = self._resolve_attribute('logdir')
        if log_dir:
            self._make_dir(os.path.join(self.initialdir, log_dir))

    def _resolve_attribute(self, attribute):
        """

        :return:
        """
        value = self.get(attribute)
        if not value:
            return None
        resolved_value = re.sub('\$\((.*?)\)',self._resolve_attribute_match, value)
        return resolved_value

    def _resolve_attribute_match(self, match):
        """

        :param match:
        :return:
        """
        return self.get(match.group(1), match.group(0))

class NoExecutable(Exception):
    """docstring

    """
    pass
<commit_msg>updates in job.py<commit_after>'''
Created on May 23, 2014

@author: sdc50
'''
#TODO: add ability to get stats about the job (i.e. number of jobs, run time, etc.)
#TODO: add ability to submit to remote schedulers

import os, subprocess, re
from collections import OrderedDict

class Job(object):
    """classdocs

    http://research.cs.wisc.edu/htcondor/manual/v7.8/condor_submit.html#man-condor-submit

    """

    def __init__(self, name, attributes=None, executable=None, arguments=None, num_jobs=None):
        """Constructor

        """
        if attributes:
            assert isinstance(attributes, dict)
        self.__dict__['_name'] = name
        self.__dict__['_attributes'] = attributes or OrderedDict()
        self.__dict__['_num_jobs'] = 1
        self.__dict__['_cluster_id'] = None
        self.__dict__['_job_file'] = ''
        self.job_name = name
        self.executable = executable
        self.arguments = arguments

    def __str__(self):
        """docstring

        """

        return '\n'.join(self._list_attributes()) + '\n\nqueue %d\n' % (self.num_jobs)

    def __repr__(self):
        """docstring

        """
        return self.attributes.__repr__()

    def __deepcopy__(self):
        """

        :return:
        """
        from copy import deepcopy
        copy = Job(self.name)
        copy.__dict__.update(self._dict__)
        copy._attributes = deepcopy(self.attributes)

    def __getattr__(self, item):
        """

        :param item:
        :return:
        """
        self.get(item)

    def __setattr__(self, key, value):
        """

        :param key:
        :param value:
        :return:
        """
        self.set(key, value)

    @property
    def name(self):
        """

        :return:
        """
        self._name = self.get('job_name')
        return self._name

    @name.setter
    def name(self,name):
        """

        :param name:
        :return:
        """
        self.set('job_name', name)

    @property
    def attributes(self):
        """

        :return:
        """
        return self._attributes

    @property
    def num_jobs(self):
        """

        :return:
        """
        return self._num_jobs

    @num_jobs.setter
    def num_jobs(self, num_jobs):
        """

        :param num_jobs:
        :return:
        """
        self._num_jobs = int(num_jobs)

    @property
    def cluster_id(self):
        """

        :return:
        """
        return self._cluster_id

    @property
    def job_file(self):
        """

        :return:
        """
        #TODO: should the job file be just the name or the name and initdir?
        job_file_name = '%s.job' % (self.name)
        job_file_path = os.path.join(self.initialdir,job_file_name)
        self._job_file = job_file_path
        return self._job_file

    @property
    def log_file(self):
        """

        :return:
        """
        log_file = self.get('log')
        if not log_file:
            log_file = '%s.log' % (self.name)
            self.set('log', log_file)
        return self._resolve_attribute('log')

    @property
    def initialdir(self):
        """

        :return:
        """
        initial_dir = self._resolve_attribute('initialdir')
        if not initial_dir:
            initial_dir = os.getcwd()
        return initial_dir

    def submit(self, queue=None, options=None):
        """docstring

        """

        if not self.executable:
            raise NoExecutable('You cannot submit a job without an executable')

        self._num_jobs = queue or self.num_jobs

        self._write_job_file()

        args = ['condor_submit']
        if options:
            args.append(options)
        args.append(self.job_file)

        process = subprocess.Popen(args, stdout = subprocess.PIPE, stderr=subprocess.PIPE)
        out,err = process.communicate()

        if err:
            if re.match('WARNING',err):
                print(err)
            else:
                raise Exception(err)
        print out
        try:
            self._cluster_id = int(re.search('(?<=cluster |\*\* Proc )(\d*)', out).group(1))
        except:
            self._cluster_id = -1
        return self.cluster_id

    def remove(self):
        """docstring

        """
        options = self.cluster_id ##TODO allow other options (like specific processes in a cluster to be removed)
        args = ['condor_rm', options]
        process = subprocess.Popen(args, stdout = subprocess.PIPE, stderr=subprocess.PIPE)
        out,err = process.communicate()
        print(out,err)

    def edit(self):
        """interface for CLI edit commands

        """
        raise NotImplementedError("This method is not yet implemented")

    def status(self):
        """docstring

        """
        raise NotImplementedError("This method is not yet implemented")

    def wait(self):
        """

        :return:
        """
        process = subprocess.Popen(['condor_wait', self.log_file], stdout = subprocess.PIPE, stderr=subprocess.PIPE)
        process.communicate()

    def get(self, attr, value=None):
        """get attribute from job file

        """
        try:
            value = self.attributes[attr]
        except KeyError:
            pass
        return value

    def set(self, attr, value):
        """set attribute in job file

        """
        self.attributes[attr] = value

    def delete(self, attr):
        """delete attribute from job file
        :param attr:
        :return:none
        """
        self.attributes.pop(attr)

    def _write_job_file(self):
        self._make_job_dirs()
        job_file = open(self.job_file, 'w')
        job_file.write(self.__str__())
        job_file.close()

    def _list_attributes(self):
        list = []
        for k,v in self.attributes.iteritems():
            if v:
                list.append(k + ' = ' + str(v))
        return list

    def _make_dir(self, dir_name):
        """docstring

        """
        try:
            os.makedirs(dir_name)
        except OSError:
            pass

    def _make_job_dirs(self):
        """docstring

        """
        self._make_dir(self.initialdir)
        log_dir = self._resolve_attribute('logdir')
        if log_dir:
            self._make_dir(os.path.join(self.initialdir, log_dir))

    def _resolve_attribute(self, attribute):
        """

        :return:
        """
        value = self.get(attribute)
        if not value:
            return None
        resolved_value = re.sub('\$\((.*?)\)',self._resolve_attribute_match, value)
        return resolved_value

    def _resolve_attribute_match(self, match):
        """

        :param match:
        :return:
        """
        return self.get(match.group(1), match.group(0))

class NoExecutable(Exception):
    """docstring

    """
    pass
<|endoftext|>


<commit_before>#!/usr/bin/env python

from os import chdir
from os.path import isfile, isdir
import sys

from util import status

from sh import Command, make, git

import os

from multiprocessing import cpu_count
max_load = cpu_count() * 0.9

minimal = False
if '--minimal' in sys.argv:
    minimal = True

def check_llvm():
    try:
        llvm_path = './llvm/Release+Asserts'
        assert isfile(llvm_path+'/lib/ocaml/llvm.cma') # is llvm.cma there?
        llvm_config = Command(llvm_path+'/bin/llvm-config')
        ver = llvm_config('--version') # try calling llvm-config
        print ver
        assert '3.2' in ver
        return True
    except:
        return False

class OCamlVersionError(Exception):
    url = 'http://caml.inria.fr/'
    def __str__(self):
        return 'OCaml >= 3.12 is required'

class OCamlFindlibError(Exception):
    url = 'http://projects.camlcity.org/projects/findlib.html'
    def __str__(self):
        return 'OCaml findlib is required'

class OCamlLibMissingError(Exception):
    def __init__(self, name):
        self.name = name
    def __str__(self):
        return 'Missing required OCaml library: %s' % self.name

def check_ocaml():
    try:
        # Test for ocaml 3.12.*
        status('Testing for OCaml 3.12.* or greater')
        from sh import ocaml, ocamlbuild
        ver = ocaml('-version')
        print ver
        assert '3.12' in ver or '4.' in ver
    except:
        raise OCamlVersionError()

    try:
        status('Testing for ocamlfind')
        from sh import ocamlfind
        print '..OK!'
    except:
        raise OCamlFindlibError()

    def assert_lib_version(lib, ver_req):
        def lib_version(lib):
            status('Testing for %s' % lib)
            info = ocamlfind('query', '-l', lib)
            return [l.split()[-1] for l in info.splitlines() if l.startswith('version')][0]
        try:
            ver = lib_version(lib)
            ver = float( '.'.join( ver.split('.')[0:2] ) )
            assert ver >= ver_req
        except:
            raise OCamlLibMissingError('%s >= %.1f' % (lib, ver_req))

    assert_lib_version('sexplib', 7.0)
    assert_lib_version('batteries', 1.4)
    return ocamlbuild

ocamlbuild = None
platform = sys.platform.lower()
if 'linux' in platform:
    try:
        status('Testing for g++')
        from sh import which
        assert which('g++')
        print '...OK!'

        status('Testing for package libc6-dev-i386')
        assert isfile('/usr/include/x86_64-linux-gnu/gnu/stubs-32.h')
        print '...OK!'

        try:
            ocamlbuild = check_ocaml()
            print '...OK!'
        except OCamlVersionError as e:
            print 'e'

        status('Testing for package libsexplib-camlp4-dev')
        assert isdir('/usr/lib/ocaml/sexplib')
        print '...OK!'
    except:
        print 'You appear to be missing some required packages. Try:'
        print 'sudo apt-get install g++ libc6-dev-i386 ocaml libsexplib-camlp4-dev'
        sys.exit(1)

if 'darwin' in platform:
    status('Testing for a sufficiently new g++')
    gxx = Command('g++')
    ver = gxx('--version')
    try:
        assert 'g++-4.' in ver
        print '...OK!'
    except:
        print 'Your g++ compiler is missing or too old.'
        print 'Trying installing the command line tools from xcode.'
        print 'They can be found in preferences -> downloads -> command line tools.'
        print 'If that doesn\'t work, update xcode and try again.'
        sys.exit(1)
    try:
        ocamlbuild = check_ocaml()
        print '...OK!'
    except OCamlVersionError as e:
        print e
        sys.exit(1)
    except OCamlFindlibError as e:
        print e
        sys.exit(1)
    except OCamlLibMissingError as e:
        print e
        sys.exit(1)

# Submodule update/init
# TODO: make --recursive optional
status('Checking out submodules')
git('submodule', 'sync')
git('submodule', 'update', '--init', '--recursive')

# TODO: always run make -C llvm, just to make sure it's up to date. Does configure cache its settings when a reconfigure is forced?
# TODO: make install in subdir, with docs
#        requires graphviz, doxygen; target ocamlbuild to alt dir?; make clean?
# Build llvm
if check_llvm():
    status('llvm appears to be present -- skipping')
else:
    chdir('llvm')
    configure = Command('./configure')
    llvm_cfg = ['--enable-assertions', '--enable-optimized']
    if minimal:
        llvm_cfg = llvm_cfg + ['--enable-targets=host,ptx,x86_64,arm']
    else:
        llvm_cfg = llvm_cfg + ['--enable-targets=all', '--enable-docs', '--enable-doxygen']
    status('''Configuring llvm:
    %s''' % llvm_cfg)
    print configure(llvm_cfg)

    status('Building llvm')
    # print make('-j', '--load-average=%f' % max_load)
    print make('-j12')

    chdir('docs')
    if not minimal:
        make('ocamldoc', '-j', '--load-average=%f' % max_load)
    # optional: check_call('make doxygen'.split(' '))
    chdir('../..')
    assert check_llvm()

# Test building
chdir('src')
status('Test: building halide.cmxa')
print ocamlbuild('-use-ocamlfind', 'halide.cmxa')
chdir('..')

status('Building C++ bindings')
print make('-C', 'cpp_bindings', '-j1') # can be flakey with first parallel build on SSD
<commit_msg>Updated bootstrap for ubuntu 12.10<commit_after>#!/usr/bin/env python

from os import chdir
from os.path import isfile, isdir
import sys

from util import status

from sh import Command, make, git

import os

from multiprocessing import cpu_count
max_load = cpu_count() * 0.9

minimal = False
if '--minimal' in sys.argv:
    minimal = True

def check_llvm():
    try:
        llvm_path = './llvm/Release+Asserts'
        assert isfile(llvm_path+'/lib/ocaml/llvm.cma') # is llvm.cma there?
        llvm_config = Command(llvm_path+'/bin/llvm-config')
        ver = llvm_config('--version') # try calling llvm-config
        print ver
        assert '3.2' in ver
        return True
    except:
        return False

class OCamlVersionError(Exception):
    url = 'http://caml.inria.fr/'
    def __str__(self):
        return 'OCaml >= 3.12 is required'

class OCamlFindlibError(Exception):
    url = 'http://projects.camlcity.org/projects/findlib.html'
    def __str__(self):
        return 'OCaml findlib is required'

class OCamlLibMissingError(Exception):
    def __init__(self, name):
        self.name = name
    def __str__(self):
        return 'Missing required OCaml library: %s' % self.name

def check_ocaml():
    try:
        # Test for ocaml 3.12.*
        status('Testing for OCaml 3.12.* or greater')
        from sh import ocaml, ocamlbuild
        ver = ocaml('-version')
        print ver
        assert '3.12' in ver or '4.' in ver
    except:
        raise OCamlVersionError()

    try:
        status('Testing for ocamlfind')
        from sh import ocamlfind
        print '..OK!'
    except:
        raise OCamlFindlibError()

    def assert_lib_version(lib, ver_req):
        def lib_version(lib):
            status('Testing for %s' % lib)
            info = ocamlfind('query', '-l', lib)
            return [l.split()[-1] for l in info.splitlines() if l.startswith('version')][0]
        try:
            ver = lib_version(lib)
            ver = float( '.'.join( ver.split('.')[0:2] ) )
            assert ver >= ver_req
        except:
            raise OCamlLibMissingError('%s >= %.1f' % (lib, ver_req))

    assert_lib_version('sexplib', 7.0)
    # We don't actually use batteries right now
    #assert_lib_version('batteries', 1.4)
    return ocamlbuild

ocamlbuild = None
platform = sys.platform.lower()
if 'linux' in platform:
    try:
        status('Testing for g++')
        from sh import which
        assert which('g++')
        print '...OK!'

        status('Testing for package libc6-dev-i386')
        assert isfile('/usr/include/x86_64-linux-gnu/gnu/stubs-32.h')
        print '...OK!'

        try:
            ocamlbuild = check_ocaml()
            print '...OK!'
        except OCamlVersionError as e:
            print 'e'

        status('Testing for package libsexplib-camlp4-dev')
        assert isdir('/usr/lib/ocaml/sexplib')
        print '...OK!'

        status('Testing for clang')
        assert which('clang')
        print '...OK!'

    except:
        print 'You appear to be missing some required packages. Try:'
        print 'sudo apt-get install clang g++ libc6-dev-i386 ocaml libsexplib-camlp4-dev'
        sys.exit(1)

if 'darwin' in platform:
    status('Testing for a sufficiently new g++')
    gxx = Command('g++')
    ver = gxx('--version')
    try:
        assert 'g++-4.' in ver
        print '...OK!'
    except:
        print 'Your g++ compiler is missing or too old.'
        print 'Trying installing the command line tools from xcode.'
        print 'They can be found in preferences -> downloads -> command line tools.'
        print 'If that doesn\'t work, update xcode and try again.'
        sys.exit(1)
    try:
        ocamlbuild = check_ocaml()
        print '...OK!'
    except OCamlVersionError as e:
        print e
        sys.exit(1)
    except OCamlFindlibError as e:
        print e
        sys.exit(1)
    except OCamlLibMissingError as e:
        print e
        sys.exit(1)

# Submodule update/init
# TODO: make --recursive optional
status('Checking out submodules')
git('submodule', 'sync')
git('submodule', 'update', '--init', '--recursive')

# TODO: always run make -C llvm, just to make sure it's up to date. Does configure cache its settings when a reconfigure is forced?
# TODO: make install in subdir, with docs
#        requires graphviz, doxygen; target ocamlbuild to alt dir?; make clean?
# Build llvm
if check_llvm():
    status('llvm appears to be present -- skipping')
else:
    chdir('llvm')
    configure = Command('./configure')
    llvm_cfg = ['--enable-assertions', '--enable-optimized']
    if minimal:
        llvm_cfg = llvm_cfg + ['--enable-targets=host,ptx,x86_64,arm']
    else:
        llvm_cfg = llvm_cfg + ['--enable-targets=all', '--enable-docs', '--enable-doxygen']
    status('''Configuring llvm:
    %s''' % llvm_cfg)
    print configure(llvm_cfg)

    status('Building llvm')
    # print make('-j', '--load-average=%f' % max_load)
    print make('-j12')

    chdir('docs')
    if not minimal:
        make('ocamldoc', '-j', '--load-average=%f' % max_load)
    # optional: check_call('make doxygen'.split(' '))
    chdir('../..')
    assert check_llvm()

# Test building
chdir('src')
status('Test: building halide.cmxa')
print ocamlbuild('-use-ocamlfind', 'halide.cmxa')
chdir('..')

status('Building C++ bindings')
print make('-C', 'cpp_bindings', '-j1') # can be flakey with first parallel build on SSD
<|endoftext|>


<commit_before><commit_msg>Fixed a small ugly bug in akismet spam filter.<commit_after><|endoftext|>


<commit_before>##by: Shevaun Lewis
##date updated: 3/11/11
##updated by: Shayne Sloggett
## revised 5/2014 by Ilia Kurenkov

# import functionality from utility module
from util import *

def classify_line(line):
    '''Given a line (as a string), determines what type of trial this line is:
    whether it is a question, a sentence or a rejected trial.
    '''
    trial_types = {
    '2' : 's',
    '6' : 'q',
    '7' : 'q',
    }
    item_type = line.split()[4]
    if item_type in trial_types:
        return trial_types[item_type]
    return 'reject'

def parse_da1_file(file_name):
    '''Given a file name extracts the subject number from it as well as all the
    sentence, question and rejected items for this subject.
    Returns a tuple of the form:
    (subject_number, sentence_list, question_list, rejected_list)
    '''
    subj_number = get_subj_num(file_name)
    with open(file_name) as da1file:
        # we start by filtering out all empty lines
        non_empty = [line for line in da1file if line]
        # we then collect the sentence, question and rejected lists
        sentences = [line for line in non_empty if classify_line(line) is 's']
        questions = [line for line in non_empty if classify_line(line) is 'q']
        rejects = [line for line in non_empty if classify_line(line) is 'reject']
    return (subj_number, sentences, questions, rejects)

def sort_da1_data(data_dir):
    file_list = gen_file_paths(data_dir, filter_func=is_DA1_file)
    return [parse_da1_file(f_name) for f_name in file_list]

def create_folder(root_path, study_exp_name, suffix, data):
    out_root = os.path.join(root_path, study_exp_name + suffix)
    os.makedirs(out_root, exist_ok=True)
    existing_data = (item for item in data if len(item[1]) > 0)
    for subj_n, sents in existing_data:
        subj_file_name = subj_n + '-' + study_exp_name + suffix + '.da1'
        file_path = os.path.join(out_root, subj_file_name)
        # write_to_txt(file_path, sents, AddNewLines=True)
        write_to_txt(file_path, sents)

def write_da1(study_exp_name, data, nest_under=None):
    suffixes = [
    ('-s', 1),
    ('-q', 2),
    ('-reject', 3)
    ]
    if nest_under:
        root_path = os.path.join(nest_under, study_exp_name + '-sorted')
    else:
        root_path = study_exp_name + '-sorted'
    for suff, index in suffixes:
        relevant = ((item[0], item[index]) for item in data)
        create_folder(root_path, study_exp_name, suff, relevant)

def get_study_name(dir_name):
    normed_dir = os.path.normpath(dir_name)
    return os.path.basename(normed_dir).split('-')[0]

def load_da1_file(file_path, index):
    subj_number = get_subj_num(file_path)
    starter = (subj_number, [], [], [])
    with open(file_path) as da1file:
        subj_items = [line for line in da1file]
    return starter[:index] + (subj_items,) + starter[index + 1:]

def load_sorted_da1(sorted_dir):
    # consider passing this around as an argument to change program's overall behavior
    suffixes = [
    ('-s', 1),
    ('-q', 2),
    ('-reject', 3)
    ]
    # think about making following line a bit simpler or clearer as to its purpose
    study_name = get_study_name(sorted_dir)
    sorted_da1 = []
    for suffix, index in suffixes:
        suff_root = os.path.join(sorted_dir, study_name + suffix)
        subj_files = gen_file_paths(suff_root)
        sorted_da1 += [load_da1_file(f, index) for f in subj_files]
    return sorted_da1

def condition_filter(ranges):
    start, total = int(ranges[0]), int(ranges[1])
    return [str(n) for n in range(start, start + total)]

def by_condition(item, cond_range):
    s_number, sents, questions, rejects = item
    exp_sents = [s for s in sents if s.split()[1] in cond_range]
    exp_qs =  [q for q in questions if q.split()[1] in cond_range]
    exp_rej =  [r for r in rejects if r.split()[1] in cond_range]
    return (s_number, exp_sents, exp_qs, exp_rej)

gen_q = '''
Do you need to split the DA1s into -s, -q, and -reject files?
(Answer "no" if you already have the data split)
'''
by_exp_question = '''
Do you want to split the data by experiment?
'''
by_exp_question2 = '''
Do you want to split the data by more experiments?
'''

def main():
    split_study = ask_unique_question(gen_q)
    # data_dir = ask the user for it
    # same for study name
    if is_yes(split_study):
        questions = [
        'folder with unsorted DA1 files',
        'name of your study',
        ]
        [da1_folder, study_name] = ask_user_questions(questions, return_list=True)
        sorted_da1s = sort_da1_data(da1_folder)
        write_da1(study_name, sorted_da1s)
    else:
        da1_folder = ask_unique_question('Enter sorted files folder:\n')
        sorted_da1s = load_sorted_da1(da1_folder)
        study_name = da1_folder

    experiment_meta_qs = [
    'Experiment name',
    'First condition number',
    'Total number of conditions'
    ]

    experiment_decision = ask_unique_question(by_exp_question)
    splitting_by_experiment = is_yes(experiment_decision)

    while splitting_by_experiment:
        exp_meta_data = ask_user_questions(experiment_meta_qs, return_list=True)
        exp_filter = condition_filter(exp_meta_data[1:])
        exp_data = (by_condition(item, exp_filter) for item in sorted_da1s)
        # exp_path = os.path.join(study_name, exp_meta_data[0])
        exp_path = exp_meta_data[0]
        write_da1(exp_path, exp_data, nest_under=da1_folder)
        continue_decision = ask_unique_question(by_exp_question2)
        splitting_by_experiment = is_yes(continue_decision)

if __name__ == '__main__':
    main()<commit_msg>cleaned up main<commit_after>##by: Shevaun Lewis
##date updated: 3/11/11
##updated by: Shayne Sloggett
## revised 5/2014 by Ilia Kurenkov

# import functionality from utility module
from util import *

def classify_line(line):
    '''Given a line (as a string), determines what type of trial this line is:
    whether it is a question, a sentence or a rejected trial.
    '''
    trial_types = {
    '2' : 's',
    '6' : 'q',
    '7' : 'q',
    }
    item_type = line.split()[4]
    if item_type in trial_types:
        return trial_types[item_type]
    return 'reject'

def parse_da1_file(file_name):
    '''Given a file name extracts the subject number from it as well as all the
    sentence, question and rejected items for this subject.
    Returns a tuple of the form:
    (subject_number, sentence_list, question_list, rejected_list)
    '''
    subj_number = get_subj_num(file_name)
    with open(file_name) as da1file:
        # we start by filtering out all empty lines
        non_empty = [line for line in da1file if line]
        # we then collect the sentence, question and rejected lists
        sentences = [line for line in non_empty if classify_line(line) is 's']
        questions = [line for line in non_empty if classify_line(line) is 'q']
        rejects = [line for line in non_empty if classify_line(line) is 'reject']
    return (subj_number, sentences, questions, rejects)

def sort_da1_data(data_dir):
    '''Sorts a folder with DA1 files.
    Given a folder name, loops over all DA1 files in it and for each file
    creates a tuple with the subject number tied to the corresponding sentence,
    question and rejected trials.
    '''
    file_list = gen_file_paths(data_dir, filter_func=is_DA1_file)
    return [parse_da1_file(file_name) for file_name in file_list]

def create_folder(root_path, study_exp_name, suffix, data):
    out_root = os.path.join(root_path, study_exp_name + suffix)
    os.makedirs(out_root, exist_ok=True)
    existing_data = (item for item in data if len(item[1]) > 0)
    for subj_n, sents in existing_data:
        subj_file_name = subj_n + '-' + study_exp_name + suffix + '.da1'
        file_path = os.path.join(out_root, subj_file_name)
        write_to_txt(file_path, sents)

def write_da1(study_exp_name, data, nest_under=None):
    suffixes = [
    ('-s', 1),
    ('-q', 2),
    ('-reject', 3)
    ]
    if nest_under:
        root_path = os.path.join(nest_under, study_exp_name + '-sorted')
    else:
        root_path = study_exp_name + '-sorted'
    for suff, index in suffixes:
        relevant = ((item[0], item[index]) for item in data)
        create_folder(root_path, study_exp_name, suff, relevant)

def get_study_name(dir_name):
    normed_dir = os.path.normpath(dir_name)
    return os.path.basename(normed_dir).split('-')[0]

def load_da1_file(file_path, index):
    subj_number = get_subj_num(file_path)
    starter = (subj_number, [], [], [])
    with open(file_path) as da1file:
        subj_items = [line for line in da1file]
    return starter[:index] + (subj_items,) + starter[index + 1:]

def load_sorted_da1(sorted_dir):
    # consider passing this around as an argument to change program's overall behavior
    suffixes = [
    ('-s', 1),
    ('-q', 2),
    ('-reject', 3)
    ]
    # think about making following line a bit simpler or clearer as to its purpose
    study_name = get_study_name(sorted_dir)
    sorted_da1 = []
    for suffix, index in suffixes:
        suff_root = os.path.join(sorted_dir, study_name + suffix)
        subj_files = gen_file_paths(suff_root)
        sorted_da1 += [load_da1_file(f, index) for f in subj_files]
    return sorted_da1

def condition_filter(ranges):
    start, total = int(ranges[0]), int(ranges[1])
    return [str(n) for n in range(start, start + total)]

def get_exp_items(item, cond_range):
    s_number, sents, questions, rejects = item
    exp_sents = [s for s in sents if s.split()[1] in cond_range]
    exp_qs =  [q for q in questions if q.split()[1] in cond_range]
    exp_rej =  [r for r in rejects if r.split()[1] in cond_range]
    return (s_number, exp_sents, exp_qs, exp_rej)

###############################################################################
## Now we can run the code

SPLIT_WHOLE_STUDY = '''
Do you need to split the DA1s into -s, -q, and -reject files?
(Answer *no* if you already have the data split.)
'''
START_EXP_SPLIT = '''
Do you want to split the data by experiment?
'''
MORE_EXP_SPLIT = '''
Do you want to split the data by more experiments?
'''

def main():
    split_study = ask_single_question(SPLIT_WHOLE_STUDY)
    if is_yes(split_study):
        questions = [
        'folder with unsorted DA1 files',
        'name of your study',
        ]
        [da1_folder, study_name] = ask_user_questions(questions, return_list=True)
        sorted_da1s = sort_da1_data(da1_folder)
        write_da1(study_name, sorted_da1s)
    else:
        da1_folder = ask_single_question('Enter sorted files folder:\n')
        sorted_da1s = load_sorted_da1(da1_folder)

    experiment_meta_qs = [
    'experiment name',
    'first condition number',
    'total number of conditions'
    ]

    experiment_split_decision = ask_single_question(START_EXP_SPLIT)
    splitting_by_experiment = is_yes(experiment_split_decision)

    while splitting_by_experiment:
        exp_meta_data = ask_user_questions(experiment_meta_qs, return_list=True)
        exp_filter = condition_filter(exp_meta_data[1:])
        exp_data = (get_exp_items(item, exp_filter) for item in sorted_da1s)
        exp_path = exp_meta_data[0]
        write_da1(exp_path, exp_data, nest_under=da1_folder)
        continue_decision = ask_single_question(MORE_EXP_SPLIT)
        splitting_by_experiment = is_yes(continue_decision)

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__version__ = '0.2'

import re

class Address(object):

    TOKEN_RE = re.compile(u'''
        (?:
            (?P<no>\d+)
            (?P<subno>之\d+)?
            (?=[巷弄號樓])
            |
            (?P<name>.+?)
        )
        (?:
            (?P<unit>[縣市鄉鎮市區村里路段街巷弄號樓])
            |
            (?=\d+(?:[之-]\d+)?[巷弄號樓]|$)
        )
    ''', re.X)

    NO    = 0
    SUBNO = 1
    NAME  = 2
    UNIT  = 3

    TO_REPLACE_RE = re.compile(u'''
    [ 　,，台-]
    |
    [０-９]
    |
    [一二三四五六七八九]?
    十?
    [一二三四五六七八九]
    (?=[段路街巷弄號樓])
    ''', re.X)

    TO_REMOVE_SET = set(u' 　,，')

    TO_REPLACE_MAP = {
        u'-' : u'之', u'台': u'臺',
        u'一': u'1', u'二': u'2', u'三': u'3', u'四': u'4', u'五': u'5',
        u'六': u'6', u'七': u'7', u'八': u'8', u'九': u'9',
    }

    @staticmethod
    def normalize(s):

        if isinstance(s, str):
            s = s.decode('utf-8')

        def replace(m):

            found = m.group()

            if found in Address.TO_REMOVE_SET:
                return u''
            if found in Address.TO_REPLACE_MAP:
                return Address.TO_REPLACE_MAP[found]

            len_found = len(found)

            # 65296 = '０'; 65305 = '９'; 65248 = '０'-'0'
            if len_found == 1 and 65296 <= ord(found) <= 65305:
                return unichr(ord(found)-65248)

            # for '十一' to '九十九'
            if len_found == 2:
                return u'1'+Address.TO_REPLACE_MAP[found[1]]
            if len_found == 3:
                return Address.TO_REPLACE_MAP[found[0]]+Address.TO_REPLACE_MAP[found[2]]

        s = Address.TO_REPLACE_RE.sub(replace, s)

        return s

    @staticmethod
    def tokenize(addr_str):
        return tuple(Address.TOKEN_RE.findall(Address.normalize(addr_str)))

    def __init__(self, addr_str):
        self.tokens = Address.tokenize(addr_str)

    @staticmethod
    def flat_tokens(tokens, n):
        # the tokens may be a set
        if n: tokens = tokens[:n]
        return u''.join(u''.join(token) for token in tokens)

    def flat(self, n=None):
        return Address.flat_tokens(self.tokens, n)

    def __repr__(self):
        return 'Address(%r)' % self.flat()

    def parse_token(self, idx):
        try:
            token = self.tokens[idx]
        except IndexError:
            return (0, 0)
        else:
            return (
                int(token[Address.NO]        or 0),
                int(token[Address.SUBNO][1:] or 0)
            )

class Rule(Address):

    RULE_TOKEN_RE = re.compile(u'''
        及以上附號|含附號以下|含附號全|含附號
        |
        以下|以上
        |
        附號全
        |
        [連至單雙全](?=[\d全]|$)
    ''', re.X)

    @staticmethod
    def parse(rule_str):

        rule_str = Address.normalize(rule_str)

        rule_tokens = set()

        def extract_token(m):

            token = m.group()
            retval = u''

            if token == u'連':
                token = u''
            elif token == u'附號全':
                retval = u'號'

            if token:
                rule_tokens.add(token)

            return retval

        addr_str = Rule.RULE_TOKEN_RE.sub(extract_token, rule_str)

        return (rule_tokens, addr_str)

    def __init__(self, rule_str):
        self.rule_tokens, addr_str = Rule.parse(rule_str)
        Address.__init__(self, addr_str)

    def flat(self, n=None, m=None):
        return Address.flat(self, n)+Address.flat_tokens(self.rule_tokens, m)

    def __repr__(self):
        return 'Rule(%r)' % self.flat()

    def match(self, addr):

        # the part reserves for rule tokens
        my_last_idx = len(self.tokens)
        my_last_idx -= bool(self.rule_tokens) and u'全' not in self.rule_tokens
        my_last_idx -= u'至' in self.rule_tokens

        his_last_idx = len(addr.tokens)
        his_last_idx -= bool(self.rule_tokens)

        my_tokens_to_match = self.tokens[:my_last_idx]
        if my_tokens_to_match:

            # the addr's tokens whose unit bigger than rule's are ignorable
            start_unit = my_tokens_to_match[0][Address.UNIT]
            for his_start_idx, his_token in enumerate(addr.tokens):
                if his_token[Address.UNIT] == start_unit:
                    break

            his_tokens_to_match = addr.tokens[his_start_idx:his_last_idx]
            if len(my_tokens_to_match) != len(his_tokens_to_match):
                return False

            for my_token, his_token in zip(my_tokens_to_match, his_tokens_to_match):
                if my_token != his_token:
                    return False

        # check the rule tokens
        his_no_pair     = addr.parse_token(-1)
        my_no_pair      = self.parse_token(-1)
        my_asst_no_pair = self.parse_token(-2)
        for rt in self.rule_tokens:
            if (
                (rt == u'全'         and not his_no_pair > (0, 0)) or
                (rt == u'單'         and not his_no_pair[0] & 1 == 1) or
                (rt == u'雙'         and not his_no_pair[0] & 1 == 0) or
                (rt == u'以上'       and not his_no_pair >= my_no_pair) or
                (rt == u'以下'       and not his_no_pair <= my_no_pair) or
                (rt == u'至'         and not (
                    my_asst_no_pair <= his_no_pair <= my_no_pair or
                    u'含附號全' in self.rule_tokens and his_no_pair[0] == my_no_pair[0]
                )) or
                (rt == u'含附號'     and not  his_no_pair[0] == my_no_pair[0]) or
                (rt == u'附號全'     and not (his_no_pair[0] == my_no_pair[0] and his_no_pair[1] > 0)) or
                (rt == u'及以上附號' and not  his_no_pair >= my_no_pair) or
                (rt == u'含附號以下' and not (his_no_pair <= my_no_pair  or his_no_pair[0] == my_no_pair[0]))
            ):
                return False

        return True

import csv
from collections import defaultdict
from itertools import izip

class Directory(object):

    @staticmethod
    def get_common_part(str_a, str_b):

        if not str_a: return str_b
        if not str_b: return str_a

        for i in range(min(len(str_a), len(str_b))):
            if str_a[i] != str_b[i]:
                break
        else:
            i += 1

        return str_a[:i]

    def __init__(self):

        # rzpair = (rule_str, zipcode)
        self.tokens_rzpairs_map = defaultdict(list)

        # gzipcode = gradual_zipcode
        self.tokens_gzipcode_map = defaultdict(str)

    def load(self, zipcode, addr_str, tail_rule_str):

        tokens = Address.tokenize(addr_str)

        # (a, b, c)
        self.tokens_rzpairs_map[tokens].append((addr_str+tail_rule_str, zipcode))

        # (a, b, c) -> (a,); (a, b); (a, b, c); (b,); (b, c); (c,)
        len_tokens = len(tokens)
        for f in range(len_tokens):
            for l in range(f, len_tokens):
                k = tokens[f:l+1]
                orig = self.tokens_gzipcode_map[k]
                comm = Directory.get_common_part(orig, zipcode)
                if comm:
                    self.tokens_gzipcode_map[k] = comm
                else:
                    del self.tokens_gzipcode_map[k]

    def load_chp_csv(self, lines, skip_first=True):

        lines_iter = iter(lines)

        if skip_first:
            next(lines_iter)

        for row in csv.reader(lines_iter):
            self.load(row[0], ''.join(row[1:-1]), row[-1])

    def find(self, addr_str):

        addr = Address(addr_str)

        for i in range(len(addr.tokens), 0, -1):

            k = addr.tokens[:i]

            rzpairs = self.tokens_rzpairs_map[k]
            if rzpairs:
                for rule_str, zipcode in rzpairs:
                    if Rule(rule_str).match(addr):
                        return zipcode

            gzipcode = self.tokens_gzipcode_map[k]
            if gzipcode:
                return gzipcode

        return ''
<commit_msg>better var names<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__version__ = '0.2'

import re

class Address(object):

    TOKEN_RE = re.compile(u'''
        (?:
            (?P<no>\d+)
            (?P<subno>之\d+)?
            (?=[巷弄號樓])
            |
            (?P<name>.+?)
        )
        (?:
            (?P<unit>[縣市鄉鎮市區村里路段街巷弄號樓])
            |
            (?=\d+(?:[之-]\d+)?[巷弄號樓]|$)
        )
    ''', re.X)

    NO    = 0
    SUBNO = 1
    NAME  = 2
    UNIT  = 3

    TO_REPLACE_RE = re.compile(u'''
    [ 　,，台-]
    |
    [０-９]
    |
    [一二三四五六七八九]?
    十?
    [一二三四五六七八九]
    (?=[段路街巷弄號樓])
    ''', re.X)

    TO_REMOVE_SET = set(u' 　,，')

    TO_REPLACE_MAP = {
        u'-' : u'之', u'台': u'臺',
        u'一': u'1', u'二': u'2', u'三': u'3', u'四': u'4', u'五': u'5',
        u'六': u'6', u'七': u'7', u'八': u'8', u'九': u'9',
    }

    @staticmethod
    def normalize(s):

        if isinstance(s, str):
            s = s.decode('utf-8')

        def replace(m):

            found = m.group()

            if found in Address.TO_REMOVE_SET:
                return u''
            if found in Address.TO_REPLACE_MAP:
                return Address.TO_REPLACE_MAP[found]

            len_found = len(found)

            # 65296 = '０'; 65305 = '９'; 65248 = '０'-'0'
            if len_found == 1 and 65296 <= ord(found) <= 65305:
                return unichr(ord(found)-65248)

            # for '十一' to '九十九'
            if len_found == 2:
                return u'1'+Address.TO_REPLACE_MAP[found[1]]
            if len_found == 3:
                return Address.TO_REPLACE_MAP[found[0]]+Address.TO_REPLACE_MAP[found[2]]

        s = Address.TO_REPLACE_RE.sub(replace, s)

        return s

    @staticmethod
    def tokenize(addr_str):
        return tuple(Address.TOKEN_RE.findall(Address.normalize(addr_str)))

    def __init__(self, addr_str):
        self.tokens = Address.tokenize(addr_str)

    @staticmethod
    def flat_tokens(tokens, n):
        # the tokens may be a set
        if n: tokens = tokens[:n]
        return u''.join(u''.join(token) for token in tokens)

    def flat(self, n=None):
        return Address.flat_tokens(self.tokens, n)

    def __repr__(self):
        return 'Address(%r)' % self.flat()

    def parse_token(self, idx):
        try:
            token = self.tokens[idx]
        except IndexError:
            return (0, 0)
        else:
            return (
                int(token[Address.NO]        or 0),
                int(token[Address.SUBNO][1:] or 0)
            )

class Rule(Address):

    RULE_TOKEN_RE = re.compile(u'''
        及以上附號|含附號以下|含附號全|含附號
        |
        以下|以上
        |
        附號全
        |
        [連至單雙全](?=[\d全]|$)
    ''', re.X)

    @staticmethod
    def parse(rule_str):

        rule_str = Address.normalize(rule_str)

        rule_tokens = set()

        def extract_token(m):

            token = m.group()
            retval = u''

            if token == u'連':
                token = u''
            elif token == u'附號全':
                retval = u'號'

            if token:
                rule_tokens.add(token)

            return retval

        addr_str = Rule.RULE_TOKEN_RE.sub(extract_token, rule_str)

        return (rule_tokens, addr_str)

    def __init__(self, rule_str):
        self.rule_tokens, addr_str = Rule.parse(rule_str)
        Address.__init__(self, addr_str)

    def flat(self, n=None, m=None):
        return Address.flat(self, n)+Address.flat_tokens(self.rule_tokens, m)

    def __repr__(self):
        return 'Rule(%r)' % self.flat()

    def match(self, addr):

        # the part reserves for rule tokens
        my_end_pos = len(self.tokens)
        my_end_pos -= bool(self.rule_tokens) and u'全' not in self.rule_tokens
        my_end_pos -= u'至' in self.rule_tokens

        his_end_pos = len(addr.tokens)
        his_end_pos -= bool(self.rule_tokens)

        my_tokens_to_match = self.tokens[:my_end_pos]
        if my_tokens_to_match:

            # the addr's tokens whose unit bigger than rule's are ignorable
            start_unit = my_tokens_to_match[0][Address.UNIT]
            for his_start_pos, his_token in enumerate(addr.tokens):
                if his_token[Address.UNIT] == start_unit:
                    break

            his_tokens_to_match = addr.tokens[his_start_pos:his_end_pos]
            if len(my_tokens_to_match) != len(his_tokens_to_match):
                return False

            for my_token, his_token in zip(my_tokens_to_match, his_tokens_to_match):
                if my_token != his_token:
                    return False

        # check the rule tokens
        his_no_pair     = addr.parse_token(-1)
        my_no_pair      = self.parse_token(-1)
        my_asst_no_pair = self.parse_token(-2)
        for rt in self.rule_tokens:
            if (
                (rt == u'全'         and not his_no_pair > (0, 0)) or
                (rt == u'單'         and not his_no_pair[0] & 1 == 1) or
                (rt == u'雙'         and not his_no_pair[0] & 1 == 0) or
                (rt == u'以上'       and not his_no_pair >= my_no_pair) or
                (rt == u'以下'       and not his_no_pair <= my_no_pair) or
                (rt == u'至'         and not (
                    my_asst_no_pair <= his_no_pair <= my_no_pair or
                    u'含附號全' in self.rule_tokens and his_no_pair[0] == my_no_pair[0]
                )) or
                (rt == u'含附號'     and not  his_no_pair[0] == my_no_pair[0]) or
                (rt == u'附號全'     and not (his_no_pair[0] == my_no_pair[0] and his_no_pair[1] > 0)) or
                (rt == u'及以上附號' and not  his_no_pair >= my_no_pair) or
                (rt == u'含附號以下' and not (his_no_pair <= my_no_pair  or his_no_pair[0] == my_no_pair[0]))
            ):
                return False

        return True

import csv
from collections import defaultdict
from itertools import izip

class Directory(object):

    @staticmethod
    def get_common_part(str_a, str_b):

        if not str_a: return str_b
        if not str_b: return str_a

        for i in range(min(len(str_a), len(str_b))):
            if str_a[i] != str_b[i]:
                break
        else:
            i += 1

        return str_a[:i]

    def __init__(self):

        # rzpair = (rule_str, zipcode)
        self.tokens_rzpairs_map = defaultdict(list)

        # gzipcode = gradual_zipcode
        self.tokens_gzipcode_map = defaultdict(str)

    def load(self, zipcode, addr_str, tail_rule_str):

        tokens = Address.tokenize(addr_str)

        # (a, b, c)
        self.tokens_rzpairs_map[tokens].append((addr_str+tail_rule_str, zipcode))

        # (a, b, c) -> (a,); (a, b); (a, b, c); (b,); (b, c); (c,)
        len_tokens = len(tokens)
        for f in range(len_tokens):
            for l in range(f, len_tokens):
                k = tokens[f:l+1]
                orig = self.tokens_gzipcode_map[k]
                comm = Directory.get_common_part(orig, zipcode)
                if comm:
                    self.tokens_gzipcode_map[k] = comm
                else:
                    del self.tokens_gzipcode_map[k]

    def load_chp_csv(self, lines, skip_first=True):

        lines_iter = iter(lines)

        if skip_first:
            next(lines_iter)

        for row in csv.reader(lines_iter):
            self.load(row[0], ''.join(row[1:-1]), row[-1])

    def find(self, addr_str):

        addr = Address(addr_str)

        for i in range(len(addr.tokens), 0, -1):

            k = addr.tokens[:i]

            rzpairs = self.tokens_rzpairs_map[k]
            if rzpairs:
                for rule_str, zipcode in rzpairs:
                    if Rule(rule_str).match(addr):
                        return zipcode

            gzipcode = self.tokens_gzipcode_map[k]
            if gzipcode:
                return gzipcode

        return ''
<|endoftext|>


<commit_before><commit_msg>Update validatorBlockNames for independentMetric<commit_after><|endoftext|>


<commit_before># Copyright (c) 2013, Martin Herkt
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.

bl_info = {
    "name": "Export: Adobe After Effects 6.0 Keyframe Data",
    "description": "Export motion tracking markers to Adobe After Effects 6.0 compatible files",
    "author": "Martin Herkt",
    "version": (0, 1, 3),
    "blender": (2, 62, 0),
    "location": "File > Export > Adobe After Effects 6.0 Keyframe Data",
    "warning": "",
    "category": "Import-Export",
    }

import bpy, mathutils, math

def write_files(prefix, context):
    scene = context.scene
    fps = scene.render.fps / scene.render.fps_base

    clipno = 0

    for clip in bpy.data.movieclips:
        trackno = 0

        for track in clip.tracking.tracks:
            with open("{0}_c{1:02d}_t{2:02d}.txt".format(prefix, clipno, trackno), "w") as f:

                frameno = clip.frame_start
                startarea = None
                startwidth = None
                startheight = None
                startrot = None

                data = []

                f.write("Adobe After Effects 6.0 Keyframe Data\r\n\r\n")
                f.write("\tUnits Per Second\t{0:.3f}\r\n".format(fps))
                f.write("\tSource Width\t{0}\r\n".format(clip.size[0]))
                f.write("\tSource Height\t{0}\r\n".format(clip.size[1]))
                f.write("\tSource Pixel Aspect Ratio\t1\r\n")
                f.write("\tComp Pixel Aspect Ratio\t1\r\n\r\n")

                while frameno <= clip.frame_duration:
                    marker = track.markers.find_frame(frameno)
                    frameno += 1

                    if not marker or marker.mute:
                        continue

                    coords = marker.co
                    corners = marker.pattern_corners

                    area = 0
                    width = math.sqrt((corners[1][0] - corners[0][0]) * (corners[1][0] - corners[0][0]) + (corners[1][1] - corners[0][1]) * (corners[1][1] - corners[0][1]))
                    height = math.sqrt((corners[3][0] - corners[0][0]) * (corners[3][0] - corners[0][0]) + (corners[3][1] - corners[0][1]) * (corners[3][1] - corners[0][1]))
                    for i in range(1,3):
                        x1 = corners[i][0] - corners[0][0]
                        y1 = corners[i][1] - corners[0][1]
                        x2 = corners[i+1][0] - corners[0][0]
                        y2 = corners[i+1][1] - corners[0][1]
                        area += x1 * y2 - x2 * y1

                    area = abs(area / 2)

                    if startarea == None:
                        startarea = area

                    if startwidth == None:
                        startwidth = width
                    if startheight == None:
                        startheight = height

                    zoom = math.sqrt(area / startarea) * 100

                    xscale = width / startwidth * 100
                    yscale = height / startheight * 100

                    p1 = mathutils.Vector(corners[0])
                    p2 = mathutils.Vector(corners[1])
                    mid = (p1 + p2) / 2
                    diff = mid - mathutils.Vector((0,0))

                    rotation = math.atan2(diff[0], diff[1]) * 180 / math.pi

                    if startrot == None:
                        startrot = rotation
                        rotation = 0
                    else:
                        rotation -= startrot - 360

                    x = coords[0] * clip.size[0]
                    y = (1 - coords[1]) * clip.size[1]

                    data.append([marker.frame, x, y, xscale, yscale, rotation])

                posline = "\t{0}\t{1:.3f}\t{2:.3f}\t0"
                scaleline = "\t{0}\t{1:.3f}\t{2:.3f}\t100"
                rotline = "\t{0}\t{1:.3f}"

                positions = "\r\n".join([posline.format(d[0], d[1], d[2]) for d in data]) + "\r\n\r\n"
                scales = "\r\n".join([scaleline.format(d[0], d[3], d[4]) for d in data]) + "\r\n\r\n"
                rotations = "\r\n".join([rotline.format(d[0], d[5]) for d in data]) + "\r\n\r\n"

                f.write("Anchor Point\r\n")
                f.write("\tFrame\tX pixels\tY pixels\tZ pixels\r\n")
                f.write(positions)

                f.write("Position\r\n")
                f.write("\tFrame\tX pixels\tY pixels\tZ pixels\r\n")
                f.write(positions)

                f.write("Scale\r\n")
                f.write("\tFrame\tX percent\tY percent\tZ percent\r\n")
                f.write(scales)

                f.write("Rotation\r\n")
                f.write("\tFrame Degrees\r\n")
                f.write(rotations)

                f.write("End of Keyframe Data\r\n")

                trackno += 1

            clipno += 1
    return {'FINISHED'}

from bpy_extras.io_utils import ExportHelper
from bpy.props import StringProperty

class ExportAFXKey(bpy.types.Operator, ExportHelper):
    """Export motion tracking markers to Adobe After Effects 6.0 compatible files"""
    bl_idname = "export.afxkey"
    bl_label = "Export to Adobe After Effects 6.0 Keyframe Data"
    filename_ext = ""
    filter_glob = StringProperty(default="*", options={'HIDDEN'})

    def execute(self, context):
        return write_files(self.filepath, context)

def menu_func(self, context):
    self.layout.operator(ExportAFXKey.bl_idname, text="Adobe After Effects 6.0 Keyframe Data")

def register():
    bpy.utils.register_class(ExportAFXKey)
    bpy.types.INFO_MT_file_export.append(menu_func)

def unregister():
    bpy.utils.unregister_class(ExportAFXKey)
    bpy.types.INFO_MT_file_export.remove(menu_func)

if __name__ == "__main__":
    register()
<commit_msg>Update to work on blender 2.80<commit_after># Copyright (c) 2013, Martin Herkt
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.

bl_info = {
    "name": "Export: Adobe After Effects 6.0 Keyframe Data",
    "description": "Export motion tracking markers to Adobe After Effects 6.0 compatible files",
    "author": "Martin Herkt",
    "version": (0, 1, 4),
    "blender": (2, 80, 0),
    "location": "File > Export > Adobe After Effects 6.0 Keyframe Data",
    "warning": "",
    "category": "Import-Export",
    }

import bpy, mathutils, math

def write_files(prefix, context):
    scene = context.scene
    fps = scene.render.fps / scene.render.fps_base

    clipno = 0

    for clip in bpy.data.movieclips:
        trackno = 0

        for track in clip.tracking.tracks:
            with open("{0}_c{1:02d}_t{2:02d}.txt".format(prefix, clipno, trackno), "w") as f:

                frameno = clip.frame_start
                startarea = None
                startwidth = None
                startheight = None
                startrot = None

                data = []

                f.write("Adobe After Effects 6.0 Keyframe Data\r\n\r\n")
                f.write("\tUnits Per Second\t{0:.3f}\r\n".format(fps))
                f.write("\tSource Width\t{0}\r\n".format(clip.size[0]))
                f.write("\tSource Height\t{0}\r\n".format(clip.size[1]))
                f.write("\tSource Pixel Aspect Ratio\t1\r\n")
                f.write("\tComp Pixel Aspect Ratio\t1\r\n\r\n")

                while frameno <= clip.frame_duration:
                    marker = track.markers.find_frame(frameno)
                    frameno += 1

                    if not marker or marker.mute:
                        continue

                    coords = marker.co
                    corners = marker.pattern_corners

                    area = 0
                    width = math.sqrt((corners[1][0] - corners[0][0]) * (corners[1][0] - corners[0][0]) + (corners[1][1] - corners[0][1]) * (corners[1][1] - corners[0][1]))
                    height = math.sqrt((corners[3][0] - corners[0][0]) * (corners[3][0] - corners[0][0]) + (corners[3][1] - corners[0][1]) * (corners[3][1] - corners[0][1]))
                    for i in range(1,3):
                        x1 = corners[i][0] - corners[0][0]
                        y1 = corners[i][1] - corners[0][1]
                        x2 = corners[i+1][0] - corners[0][0]
                        y2 = corners[i+1][1] - corners[0][1]
                        area += x1 * y2 - x2 * y1

                    area = abs(area / 2)

                    if startarea == None:
                        startarea = area

                    if startwidth == None:
                        startwidth = width
                    if startheight == None:
                        startheight = height

                    zoom = math.sqrt(area / startarea) * 100

                    xscale = width / startwidth * 100
                    yscale = height / startheight * 100

                    p1 = mathutils.Vector(corners[0])
                    p2 = mathutils.Vector(corners[1])
                    mid = (p1 + p2) / 2
                    diff = mid - mathutils.Vector((0,0))

                    rotation = math.atan2(diff[0], diff[1]) * 180 / math.pi

                    if startrot == None:
                        startrot = rotation
                        rotation = 0
                    else:
                        rotation -= startrot - 360

                    x = coords[0] * clip.size[0]
                    y = (1 - coords[1]) * clip.size[1]

                    data.append([marker.frame, x, y, xscale, yscale, rotation])

                posline = "\t{0}\t{1:.3f}\t{2:.3f}\t0"
                scaleline = "\t{0}\t{1:.3f}\t{2:.3f}\t100"
                rotline = "\t{0}\t{1:.3f}"

                positions = "\r\n".join([posline.format(d[0], d[1], d[2]) for d in data]) + "\r\n\r\n"
                scales = "\r\n".join([scaleline.format(d[0], d[3], d[4]) for d in data]) + "\r\n\r\n"
                rotations = "\r\n".join([rotline.format(d[0], d[5]) for d in data]) + "\r\n\r\n"

                f.write("Anchor Point\r\n")
                f.write("\tFrame\tX pixels\tY pixels\tZ pixels\r\n")
                f.write(positions)

                f.write("Position\r\n")
                f.write("\tFrame\tX pixels\tY pixels\tZ pixels\r\n")
                f.write(positions)

                f.write("Scale\r\n")
                f.write("\tFrame\tX percent\tY percent\tZ percent\r\n")
                f.write(scales)

                f.write("Rotation\r\n")
                f.write("\tFrame Degrees\r\n")
                f.write(rotations)

                f.write("End of Keyframe Data\r\n")

                trackno += 1

            clipno += 1
    return {'FINISHED'}

from bpy_extras.io_utils import ExportHelper
from bpy.props import StringProperty

class ExportAFXKey(bpy.types.Operator, ExportHelper):
    """Export motion tracking markers to Adobe After Effects 6.0 compatible files"""
    bl_idname = "export.afxkey"
    bl_label = "Export to Adobe After Effects 6.0 Keyframe Data"
    filename_ext = ""
    filter_glob = StringProperty(default="*", options={'HIDDEN'})

    def execute(self, context):
        return write_files(self.filepath, context)

classes = (
    ExportAFXKey,
)

def menu_func_export(self, context):
    self.layout.operator(ExportAFXKey.bl_idname, text="Adobe After Effects 6.0 Keyframe Data")

def register():
    from bpy.utils import register_class
    for cls in classes:
        register_class(cls)
    bpy.types.TOPBAR_MT_file_export.append(menu_func_export)

def unregister():
    from bpy.utils import unregister_class
    for cls in reversed(classes):
        unregister_class(cls)
    bpy.types.TOPBAR_MT_file_exportremove(menu_func_export)

if __name__ == "__main__":
    register()
<|endoftext|>


<commit_before># encoding:UTF-8
import logging

from chariot.influx import influx
from datetime import datetime
from django.core.urlresolvers import reverse
from django.http import JsonResponse, HttpResponse, StreamingHttpResponse
from django.shortcuts import get_object_or_404
import json
from django.views.generic import CreateView
from rest_framework import status, serializers, authentication
from rest_framework.authtoken.models import Token
from rest_framework.generics import ListAPIView
from rest_framework.response import Response
from rest_framework.views import APIView

from chariot import utils
from chariot.mixins import LoginRequiredMixin, BackButtonMixin
from chariot.utils import LOCALE_DATE_FMT
from deployments.models import Deployment, DeploymentSensor
from graphs.views import generate_data, last
from hubs.forms import HubCreateForm
from hubs.models import Hub
from sensors.models import Sensor
from sensors.serializers import SensorIDSerializer

logger = logging.getLogger(__name__)

class DeploymentSensorIDSerializer(serializers.ModelSerializer):
    sensor = SensorIDSerializer()

    class Meta:
        model = DeploymentSensor
        fields = ('sensor', 'cost', 'location', 'nearest_thermostat', 'room_height', 'room_area')

class DeploymentSerializer(serializers.ModelSerializer):
    hub = serializers.PrimaryKeyRelatedField(read_only=True)
    sensors = DeploymentSensorIDSerializer(many=True)

    class Meta:
        model = Deployment
        fields = ('id', 'hub', 'sensors',
                  'boiler_manufacturer', 'boiler_model', 'boiler_output', 'boiler_efficiency',
                  'boiler_type', 'boiler_thermostat', 'building_area', 'building_height')

class DeploymentListView(ListAPIView):
    model = Deployment
    serializer_class = DeploymentSerializer
    authentication_classes = (authentication.TokenAuthentication,)

    def get_queryset(self):
        return Deployment.objects.filter(end_date__isnull=True)

class DeploymentView(APIView):
    authentication_classes = (authentication.TokenAuthentication,)

    def get_object(self, pk):
        try:
            return Deployment.objects.get(pk=pk)
        except Deployment.DoesNotExist:
            raise HttpResponse(status=404)

    def get(self, request, pk, format=None):
        deployment = self.get_object(pk)
        serializer = DeploymentSerializer(deployment)
        return Response(serializer.data)

def get_token(request, id):
    try:
        hub = Hub.objects.get(id=id)
        token = Token.objects.latest('created')
        return JsonResponse({'token': token.key, 'deployment': hub.deployment.pk})
    except Hub.DoesNotExist:
        return HttpResponse(status=404)
    except Token.DoesNotExist:
        return HttpResponse(status=404)

class HubPingView(APIView):
    authentication_classes = (authentication.TokenAuthentication,)

    def put(self, request, id, format=None):
        mac_address = utils.decode_mac_address(id)

        hub = get_object_or_404(Hub, id=mac_address)

        # May raise a permission denied
        self.check_object_permissions(self.request, hub)

        hub.ping()
        hub.save()

        return Response("success", status=status.HTTP_200_OK)

class SensorListView(ListAPIView):
    model = Sensor
    serializer_class = SensorIDSerializer
    authentication_classes = (authentication.TokenAuthentication,)

    def get_queryset(self):
        mac_address = utils.decode_mac_address(self.kwargs.get('mac_address'))

        hub = get_object_or_404(Hub, id=mac_address)

        return hub.sensors

class SensorReading(APIView):
    authentication_classes = (authentication.TokenAuthentication,)

    def post(self, request):
        hub_mac_address = utils.decode_mac_address(request.data['hub'])
        hub = Hub.objects.get(id=hub_mac_address)

        reading = {
            "measurement": request.data['channel'],
            "tags": {
                "sensor": request.data['sensor'],
                "deployment": hub.deployment.pk
            },
            "fields": {
                "value": float(request.data['value'])
            }
        }
        logger.info(reading)
        if 'timestamp' in request.data:
            reading['time'] = request.data['timestamp']
        if influx.write_points([reading]):
            return Response(reading, status=status.HTTP_201_CREATED)
        return Response("failed", status=status.HTTP_400_BAD_REQUEST)

class DeploymentPrediction(APIView):
    authentication_classes = (authentication.TokenAuthentication,)

    def put(self, request, pk):
        prediction_json = json.loads(request.body)
        for prediction in prediction_json:
            try:
                deployment_id = prediction['Deployment_Id']
                deployment = Deployment.objects.get(pk=deployment_id)
                deployment.prediction = json.dumps(prediction)
                deployment.save()
            except Deployment.DoesNotExist:
                return HttpResponse(status=404)

        return Response("success", status=status.HTTP_200_OK)

class HubCreateView(LoginRequiredMixin, BackButtonMixin, CreateView):
    form_class = HubCreateForm
    model = Hub
    template_name = 'sensors/device_create.html'

    def get_success_url(self):
        return reverse('devices')

    def get_back_url(self):
        return reverse('devices')

class LatestDataView(APIView):
    authentication_classes = (authentication.TokenAuthentication,)

    def get(self, request, pk, format=None):
        deployment = Deployment.objects.get(pk=pk)
        result = []

        for sensor in deployment.sensors.all():
            sensor_obj = {'id': sensor.id, 'channels': []}
            for channel in sensor.channels.all():
                value = last(deployment.id, sensor.id, channel.id)
                channel_obj = {'id': channel.id, 'value': value}
                sensor_obj['channels'].append(channel_obj)
            result.append(sensor_obj)

        return JsonResponse(result)

class DataView(APIView):
    authentication_classes = (authentication.TokenAuthentication,)

    def get(self, request, pk, format=None):
        aggregate = None
        sensors = None
        channels = None
        start = None
        end = None

        if 'sensors' in request.GET:
            sensors = request.GET['sensors'].split(",")

        if 'channels' in request.GET:
            if request.GET['channels'].lower() == 'all':
                channels = 'all'
            else:
                channels = request.GET['channels'].split(",")

        if 'start' in request.GET:
            start = datetime.strptime(request.GET['start'], LOCALE_DATE_FMT)

        if 'end' in request.GET:
            end = datetime.strptime(request.GET['end'], LOCALE_DATE_FMT)

        simplify = 'simplify' not in request.GET or request.GET['simplify'].lower() != 'false'
        if 'aggregate' in request.GET:
            aggregate = request.GET['aggregate']

        return StreamingHttpResponse(generate_data(pk, sensors, channels, simplify, start, end, aggregate),
                                     content_type="application/json")
<commit_msg>Fix sensor reference<commit_after># encoding:UTF-8
import logging

from chariot.influx import influx
from datetime import datetime
from django.core.urlresolvers import reverse
from django.http import JsonResponse, HttpResponse, StreamingHttpResponse
from django.shortcuts import get_object_or_404
import json
from django.views.generic import CreateView
from rest_framework import status, serializers, authentication
from rest_framework.authtoken.models import Token
from rest_framework.generics import ListAPIView
from rest_framework.response import Response
from rest_framework.views import APIView

from chariot import utils
from chariot.mixins import LoginRequiredMixin, BackButtonMixin
from chariot.utils import LOCALE_DATE_FMT
from deployments.models import Deployment, DeploymentSensor
from graphs.views import generate_data, last
from hubs.forms import HubCreateForm
from hubs.models import Hub
from sensors.models import Sensor
from sensors.serializers import SensorIDSerializer

logger = logging.getLogger(__name__)

class DeploymentSensorIDSerializer(serializers.ModelSerializer):
    sensor = SensorIDSerializer()

    class Meta:
        model = DeploymentSensor
        fields = ('sensor', 'cost', 'location', 'nearest_thermostat', 'room_height', 'room_area')

class DeploymentSerializer(serializers.ModelSerializer):
    hub = serializers.PrimaryKeyRelatedField(read_only=True)
    sensors = DeploymentSensorIDSerializer(many=True)

    class Meta:
        model = Deployment
        fields = ('id', 'hub', 'sensors',
                  'boiler_manufacturer', 'boiler_model', 'boiler_output', 'boiler_efficiency',
                  'boiler_type', 'boiler_thermostat', 'building_area', 'building_height')

class DeploymentListView(ListAPIView):
    model = Deployment
    serializer_class = DeploymentSerializer
    authentication_classes = (authentication.TokenAuthentication,)

    def get_queryset(self):
        return Deployment.objects.filter(end_date__isnull=True)

class DeploymentView(APIView):
    authentication_classes = (authentication.TokenAuthentication,)

    def get_object(self, pk):
        try:
            return Deployment.objects.get(pk=pk)
        except Deployment.DoesNotExist:
            raise HttpResponse(status=404)

    def get(self, request, pk, format=None):
        deployment = self.get_object(pk)
        serializer = DeploymentSerializer(deployment)
        return Response(serializer.data)

def get_token(request, id):
    try:
        hub = Hub.objects.get(id=id)
        token = Token.objects.latest('created')
        return JsonResponse({'token': token.key, 'deployment': hub.deployment.pk})
    except Hub.DoesNotExist:
        return HttpResponse(status=404)
    except Token.DoesNotExist:
        return HttpResponse(status=404)

class HubPingView(APIView):
    authentication_classes = (authentication.TokenAuthentication,)

    def put(self, request, id, format=None):
        mac_address = utils.decode_mac_address(id)

        hub = get_object_or_404(Hub, id=mac_address)

        # May raise a permission denied
        self.check_object_permissions(self.request, hub)

        hub.ping()
        hub.save()

        return Response("success", status=status.HTTP_200_OK)

class SensorListView(ListAPIView):
    model = Sensor
    serializer_class = SensorIDSerializer
    authentication_classes = (authentication.TokenAuthentication,)

    def get_queryset(self):
        mac_address = utils.decode_mac_address(self.kwargs.get('mac_address'))

        hub = get_object_or_404(Hub, id=mac_address)

        return hub.sensors

class SensorReading(APIView):
    authentication_classes = (authentication.TokenAuthentication,)

    def post(self, request):
        hub_mac_address = utils.decode_mac_address(request.data['hub'])
        hub = Hub.objects.get(id=hub_mac_address)

        reading = {
            "measurement": request.data['channel'],
            "tags": {
                "sensor": request.data['sensor'],
                "deployment": hub.deployment.pk
            },
            "fields": {
                "value": float(request.data['value'])
            }
        }
        logger.info(reading)
        if 'timestamp' in request.data:
            reading['time'] = request.data['timestamp']
        if influx.write_points([reading]):
            return Response(reading, status=status.HTTP_201_CREATED)
        return Response("failed", status=status.HTTP_400_BAD_REQUEST)

class DeploymentPrediction(APIView):
    authentication_classes = (authentication.TokenAuthentication,)

    def put(self, request, pk):
        prediction_json = json.loads(request.body)
        for prediction in prediction_json:
            try:
                deployment_id = prediction['Deployment_Id']
                deployment = Deployment.objects.get(pk=deployment_id)
                deployment.prediction = json.dumps(prediction)
                deployment.save()
            except Deployment.DoesNotExist:
                return HttpResponse(status=404)

        return Response("success", status=status.HTTP_200_OK)

class HubCreateView(LoginRequiredMixin, BackButtonMixin, CreateView):
    form_class = HubCreateForm
    model = Hub
    template_name = 'sensors/device_create.html'

    def get_success_url(self):
        return reverse('devices')

    def get_back_url(self):
        return reverse('devices')

class LatestDataView(APIView):
    authentication_classes = (authentication.TokenAuthentication,)

    def get(self, request, pk, format=None):
        deployment = Deployment.objects.get(pk=pk)
        result = []

        for sensor in deployment.sensors.all():
            sensor_obj = {'id': sensor.id, 'channels': []}
            for channel in sensor.sensor.channels.all():
                value = last(deployment.id, sensor.id, channel.id)
                channel_obj = {'id': channel.id, 'value': value}
                sensor_obj['channels'].append(channel_obj)
            result.append(sensor_obj)

        return JsonResponse(result)

class DataView(APIView):
    authentication_classes = (authentication.TokenAuthentication,)

    def get(self, request, pk, format=None):
        aggregate = None
        sensors = None
        channels = None
        start = None
        end = None

        if 'sensors' in request.GET:
            sensors = request.GET['sensors'].split(",")

        if 'channels' in request.GET:
            if request.GET['channels'].lower() == 'all':
                channels = 'all'
            else:
                channels = request.GET['channels'].split(",")

        if 'start' in request.GET:
            start = datetime.strptime(request.GET['start'], LOCALE_DATE_FMT)

        if 'end' in request.GET:
            end = datetime.strptime(request.GET['end'], LOCALE_DATE_FMT)

        simplify = 'simplify' not in request.GET or request.GET['simplify'].lower() != 'false'
        if 'aggregate' in request.GET:
            aggregate = request.GET['aggregate']

        return StreamingHttpResponse(generate_data(pk, sensors, channels, simplify, start, end, aggregate),
                                     content_type="application/json")
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Hive Appier Framework
# Copyright (C) 2008-2014 Hive Solutions Lda.
#
# This file is part of Hive Appier Framework.
#
# Hive Appier Framework is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Hive Appier Framework is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Hive Appier Framework. If not, see <http://www.gnu.org/licenses/>.

__author__ = "João Magalhães joamag@hive.pt>"
""" The author(s) of the module """

__version__ = "1.0.0"
""" The version of the module """

__revision__ = "$LastChangedRevision$"
""" The revision number of the module """

__date__ = "$LastChangedDate$"
""" The last change date of the module """

__copyright__ = "Copyright (c) 2008-2014 Hive Solutions Lda."
""" The copyright for the module """

__license__ = "GNU General Public License (GPL), Version 3"
""" The license for the module """

import os
import time
import uuid
import shelve
import hashlib
import datetime

from appier import config
from appier import legacy

EXPIRE_TIME = datetime.timedelta(days = 31)
""" The default expire time to be used in new sessions
in case no expire time is provided to the creation of
the session instance """

class Session(object):
    """
    Abstract session class to be used as a reference in
    the implementation of the proper classes.

    Some of the global functionality may be abstracted
    under this class and reused in the concrete classes.
    """

    def __init__(self, name = "session", expire = EXPIRE_TIME):
        object.__init__(self)
        self.sid = self._gen_sid()
        self.name = name
        self.expire = time.time() + self._to_seconds(expire)
        self.dirty = False

    def __len__(self):
        return 0

    def __getitem__(self, key):
        raise KeyError("not found")

    def __setitem__(self, key, value):
        self.mark()

    def __delitem__(self, key):
        self.mark()

    def __contains__(self, item):
        return False

    def __nonzero__(self):
        return True

    def __bool__(self):
        return True

    @classmethod
    def new(cls, *args, **kwargs):
        return cls(*args, **kwargs)

    @classmethod
    def get_s(cls, sid):
        return cls()

    @classmethod
    def expire(self, sid):
        pass

    @classmethod
    def count(cls):
        return 0

    @classmethod
    def open(cls):
        cls.gc()

    @classmethod
    def close(cls):
        pass

    @classmethod
    def gc(cls):
        pass

    def start(self):
        pass

    def flush(self):
        self.mark(dirty = False)

    def mark(self, dirty = True):
        self.dirty = dirty

    def is_expired(self):
        has_expire = hasattr(self, "expire")
        if not has_expire: return False

        current = time.time()
        return current >= self.expire

    def is_dirty(self):
        if not hasattr(self, "dirty"): return True
        return self.dirty

    def ensure(self, *args, **kwargs):
        return self

    def get(self, key, default = None):
        try: value = self.__getitem__(key)
        except KeyError: value = default
        return value

    def _gen_sid(self):
        token_s = str(uuid.uuid4())
        token_s = legacy.bytes(token_s)
        token = hashlib.sha256(token_s).hexdigest()
        return token

    def _to_seconds(self, delta):
        return (delta.microseconds +
            (delta.seconds + delta.days * 24 * 3600) * 10 ** 6) / 10 ** 6

class MockSession(Session):
    """
    Temporary mock session to be used while no final
    session is yet defined in the request, this is a
    special case of a session and should not comply
    with the pre-defined standard operations.
    """

    def __init__(self, request, name = "mock", *args, **kwargs):
        Session.__init__(self, name = name, *args, **kwargs)
        self.request = request

    def __setitem__(self, key, value):
        session = self.ensure()
        return session.__setitem__(key, value)

    def ensure(self, *args, **kwargs):
        session_c = self.request.session_c
        session = session_c.new(*args, **kwargs)
        self.request.session = session
        self.request.set_cookie = "sid=%s" % session.sid
        return session

class MemorySession(Session):

    SESSIONS = {}
    """ Global static sessions map where all the
    (in-memory) session instances are going to be
    stored to be latter retrieved """

    def __init__(self, name = "session", *args, **kwargs):
        Session.__init__(self, name = name, *args, **kwargs)
        self.data = {}
        self["sid"] = self.sid

    def __len__(self):
        return self.data.__len__()

    def __getitem__(self, key):
        return self.data.__getitem__(key)

    def __setitem__(self, key, value):
        self.mark(); return self.data.__setitem__(key, value)

    def __delitem__(self, key):
        self.mark(); return self.data.__delitem__(key)

    def __contains__(self, item):
        return self.data.__contains__(item)

    @classmethod
    def new(cls, *args, **kwargs):
        session = cls(*args, **kwargs)
        cls.SESSIONS[session.sid] = session
        return session

    @classmethod
    def get_s(cls, sid):
        session = cls.SESSIONS.get(sid, None)
        if not session: return session
        is_expired = session.is_expired()
        if is_expired: cls.expire(sid)
        session = None if is_expired else session
        return session

    @classmethod
    def expire(cls, sid):
        del cls.SESSIONS[sid]

    @classmethod
    def count(cls):
        return len(cls.SESSIONS)

class FileSession(Session):

    SHELVE = None
    """ Global shelve object reference should reflect the
    result of opening a file in shelve mode, this is a global
    object and only one instance should exist per process """

    def __init__(self, name = "file", *args, **kwargs):
        Session.__init__(self, name = name, *args, **kwargs)
        self.data = {}
        self["sid"] = self.sid

    @classmethod
    def new(cls, *args, **kwargs):
        if not cls.SHELVE: cls.open()
        session = cls(*args, **kwargs)
        cls.SHELVE[session.sid] = session
        return session

    @classmethod
    def get_s(cls, sid):
        if not cls.SHELVE: cls.open()
        session = cls.SHELVE.get(sid, None)
        if not session: return session
        is_expired = session.is_expired()
        if is_expired: cls.expire(sid)
        session = None if is_expired else session
        return session

    @classmethod
    def expire(cls, sid):
        del cls.SHELVE[sid]

    @classmethod
    def count(cls):
        return len(cls.SHELVE)

    @classmethod
    def open(cls, file_path = "session.shelve"):
        base_path = config.conf("APPIER_BASE_PATH", "")
        file_path = os.path.join(base_path, file_path)
        cls.SHELVE = shelve.open(
            file_path,
            protocol = 2,
            writeback = True
        )
        cls.gc()

    @classmethod
    def close(cls):
        cls.SHELVE.close()
        cls.SHELVE = None

    @classmethod
    def gc(cls):
        for sid in cls.SHELVE:
            session = cls.SHELVE.get(sid, None)
            is_expired = session.is_expired()
            if is_expired: cls.expire(sid)

    def __len__(self):
        return self.data.__len__()

    def __getitem__(self, key):
        return self.data.__getitem__(key)

    def __setitem__(self, key, value):
        self.mark(); return self.data.__setitem__(key, value)

    def __delitem__(self, key):
        self.mark(); return self.data.__delitem__(key)

    def __contains__(self, item):
        return self.data.__contains__(item)

    def flush(self):
        if not self.is_dirty(): return
        self.mark(dirty = False)
        cls = self.__class__
        cls.SHELVE.sync()

class RedisSession(Session):
    pass
<commit_msg>new default dirty values<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Hive Appier Framework
# Copyright (C) 2008-2014 Hive Solutions Lda.
#
# This file is part of Hive Appier Framework.
#
# Hive Appier Framework is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Hive Appier Framework is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Hive Appier Framework. If not, see <http://www.gnu.org/licenses/>.

__author__ = "João Magalhães joamag@hive.pt>"
""" The author(s) of the module """

__version__ = "1.0.0"
""" The version of the module """

__revision__ = "$LastChangedRevision$"
""" The revision number of the module """

__date__ = "$LastChangedDate$"
""" The last change date of the module """

__copyright__ = "Copyright (c) 2008-2014 Hive Solutions Lda."
""" The copyright for the module """

__license__ = "GNU General Public License (GPL), Version 3"
""" The license for the module """

import os
import time
import uuid
import shelve
import hashlib
import datetime

from appier import config
from appier import legacy

EXPIRE_TIME = datetime.timedelta(days = 31)
""" The default expire time to be used in new sessions
in case no expire time is provided to the creation of
the session instance """

class Session(object):
    """
    Abstract session class to be used as a reference in
    the implementation of the proper classes.

    Some of the global functionality may be abstracted
    under this class and reused in the concrete classes.
    """

    def __init__(self, name = "session", expire = EXPIRE_TIME):
        object.__init__(self)
        self.sid = self._gen_sid()
        self.name = name
        self.expire = time.time() + self._to_seconds(expire)
        self.dirty = True

    def __len__(self):
        return 0

    def __getitem__(self, key):
        raise KeyError("not found")

    def __setitem__(self, key, value):
        self.mark()

    def __delitem__(self, key):
        self.mark()

    def __contains__(self, item):
        return False

    def __nonzero__(self):
        return True

    def __bool__(self):
        return True

    @classmethod
    def new(cls, *args, **kwargs):
        return cls(*args, **kwargs)

    @classmethod
    def get_s(cls, sid):
        return cls()

    @classmethod
    def expire(self, sid):
        pass

    @classmethod
    def count(cls):
        return 0

    @classmethod
    def open(cls):
        cls.gc()

    @classmethod
    def close(cls):
        pass

    @classmethod
    def gc(cls):
        pass

    def start(self):
        pass

    def flush(self):
        self.mark(dirty = False)

    def mark(self, dirty = True):
        self.dirty = dirty

    def is_expired(self):
        has_expire = hasattr(self, "expire")
        if not has_expire: return False

        current = time.time()
        return current >= self.expire

    def is_dirty(self):
        if not hasattr(self, "dirty"): return True
        return self.dirty

    def ensure(self, *args, **kwargs):
        return self

    def get(self, key, default = None):
        try: value = self.__getitem__(key)
        except KeyError: value = default
        return value

    def _gen_sid(self):
        token_s = str(uuid.uuid4())
        token_s = legacy.bytes(token_s)
        token = hashlib.sha256(token_s).hexdigest()
        return token

    def _to_seconds(self, delta):
        return (delta.microseconds +
            (delta.seconds + delta.days * 24 * 3600) * 10 ** 6) / 10 ** 6

class MockSession(Session):
    """
    Temporary mock session to be used while no final
    session is yet defined in the request, this is a
    special case of a session and should not comply
    with the pre-defined standard operations.
    """

    def __init__(self, request, name = "mock", *args, **kwargs):
        Session.__init__(self, name = name, *args, **kwargs)
        self.request = request

    def __setitem__(self, key, value):
        session = self.ensure()
        return session.__setitem__(key, value)

    def ensure(self, *args, **kwargs):
        session_c = self.request.session_c
        session = session_c.new(*args, **kwargs)
        self.request.session = session
        self.request.set_cookie = "sid=%s" % session.sid
        return session

class MemorySession(Session):

    SESSIONS = {}
    """ Global static sessions map where all the
    (in-memory) session instances are going to be
    stored to be latter retrieved """

    def __init__(self, name = "session", *args, **kwargs):
        Session.__init__(self, name = name, *args, **kwargs)
        self.data = {}
        self["sid"] = self.sid

    def __len__(self):
        return self.data.__len__()

    def __getitem__(self, key):
        return self.data.__getitem__(key)

    def __setitem__(self, key, value):
        self.mark(); return self.data.__setitem__(key, value)

    def __delitem__(self, key):
        self.mark(); return self.data.__delitem__(key)

    def __contains__(self, item):
        return self.data.__contains__(item)

    @classmethod
    def new(cls, *args, **kwargs):
        session = cls(*args, **kwargs)
        cls.SESSIONS[session.sid] = session
        return session

    @classmethod
    def get_s(cls, sid):
        session = cls.SESSIONS.get(sid, None)
        if not session: return session
        is_expired = session.is_expired()
        if is_expired: cls.expire(sid)
        session = None if is_expired else session
        return session

    @classmethod
    def expire(cls, sid):
        del cls.SESSIONS[sid]

    @classmethod
    def count(cls):
        return len(cls.SESSIONS)

class FileSession(Session):

    SHELVE = None
    """ Global shelve object reference should reflect the
    result of opening a file in shelve mode, this is a global
    object and only one instance should exist per process """

    def __init__(self, name = "file", *args, **kwargs):
        Session.__init__(self, name = name, *args, **kwargs)
        self.data = {}
        self["sid"] = self.sid

    @classmethod
    def new(cls, *args, **kwargs):
        if not cls.SHELVE: cls.open()
        session = cls(*args, **kwargs)
        cls.SHELVE[session.sid] = session
        return session

    @classmethod
    def get_s(cls, sid):
        if not cls.SHELVE: cls.open()
        session = cls.SHELVE.get(sid, None)
        if not session: return session
        is_expired = session.is_expired()
        if is_expired: cls.expire(sid)
        session = None if is_expired else session
        return session

    @classmethod
    def expire(cls, sid):
        del cls.SHELVE[sid]

    @classmethod
    def count(cls):
        return len(cls.SHELVE)

    @classmethod
    def open(cls, file_path = "session.shelve"):
        base_path = config.conf("APPIER_BASE_PATH", "")
        file_path = os.path.join(base_path, file_path)
        cls.SHELVE = shelve.open(
            file_path,
            protocol = 2,
            writeback = True
        )
        cls.gc()

    @classmethod
    def close(cls):
        cls.SHELVE.close()
        cls.SHELVE = None

    @classmethod
    def gc(cls):
        for sid in cls.SHELVE:
            session = cls.SHELVE.get(sid, None)
            is_expired = session.is_expired()
            if is_expired: cls.expire(sid)

    def __len__(self):
        return self.data.__len__()

    def __getitem__(self, key):
        return self.data.__getitem__(key)

    def __setitem__(self, key, value):
        self.mark(); return self.data.__setitem__(key, value)

    def __delitem__(self, key):
        self.mark(); return self.data.__delitem__(key)

    def __contains__(self, item):
        return self.data.__contains__(item)

    def flush(self):
        if not self.is_dirty(): return
        self.mark(dirty = False)
        cls = self.__class__
        cls.SHELVE.sync()

class RedisSession(Session):
    pass
<|endoftext|>