<commit_before>#!/usr/bin/env python

import os, subprocess, argparse, re
import webbrowser
from datetime import datetime

class JarvisSettings(object):

    def __init__(self):
        # TODO: Make a test mode where it writes to a test directory.
        self._env_dir_jarvis_root = os.environ['JARVIS_DIR_ROOT']
        self._env_author = os.environ['JARVIS_AUTHOR']
        self._env_version = "0.1.0"

    @property
    def root_directory(self):
        return self._env_dir_jarvis_root

    @property
    def author(self):
        return self._env_author

    @property
    def document_version(self):
        return self._env_version

if "__main__" == __name__:
    parser = argparse.ArgumentParser(description='Jarvis is used for personal information management')

    subparsers = parser.add_subparsers(help='Actions for Jarvis',
            dest='action_name')

    # New actions
    parser_new = subparsers.add_parser('new', help='Create an information element')
    subparsers_new = parser_new.add_subparsers(help='Types of new information element',
            dest='element_type')
    parser_new_log = subparsers_new.add_parser('log', help='Create a new log entry')
    parser_new_tag = subparsers_new.add_parser('tag', help='Create a new tag element')
    parser_new_tag.add_argument('tag_name', help='Tag name')

    # Show actions
    parser_show = subparsers.add_parser('show', help='Show information elements')
    subparsers_show = parser_show.add_subparsers(help='Types of show actions',
            dest='show_type')

    parser_show_logs = subparsers_show.add_parser('logs', help='List all logs')
    parser_show_logs.add_argument('-t', '--tag', nargs='?', help='Tag to search')

    parser_show_lastlog = subparsers_show.add_parser('lastlog',
            help='Open last log entry in the browser')
    parser_show_lastlog.add_argument('-t', '--tag', nargs='?', required=True,
            help='Tag to search')

    parser_show_log = subparsers_show.add_parser('log', help='Open log entry in the browser')
    parser_show_log.add_argument('log_entry_name', help='Log name')

    parser_show_tags = subparsers_show.add_parser('tags', help='List all tags')

    parser_show_tag = subparsers_show.add_parser('tag', help='Open tag in the browser')
    parser_show_tag.add_argument('tag_name', help='Tag name')

    args = parser.parse_args()

    # NOTE: Argparse should filter and validate and ensure that only the known
    # choices get through to this point.
    # UPDATE: There is a case where the subsubparser allows for Nones to slip
    # through. The required flag doesn't seem to be reinforced so the following
    # is acceptable to Argparse which sucks for me:
    #   jarvis.py new

    js = JarvisSettings()

    if args.action_name == 'new':
        created = datetime.utcnow().replace(microsecond=0)

        def create_stub_file(subdir_name, filename):
            dir_target = "{0}/{1}".format(js.root_directory, subdir_name)

            filepath = "{0}/{1}.md".format(dir_target, filename)

            if os.path.isfile(filepath):
                raise IOError("File already exists! {0}".format(filepath))

            with open(filepath, 'w') as f:
                f.write("Author: {0}\n".format(js.author))
                f.write("Created: {0}\n".format(created.isoformat()))
                f.write("Version: {0}\n".format(js.document_version))
                f.write("Tags: \n")

            return filepath

        if args.element_type == 'log':
            # datetime.fromtimestamp(0) is not Unix epoch and returns
            # 1969-12-31 19:00 instead.
            epoch = datetime(1970, 1, 1)

            filepath = create_stub_file('LogEntries',
                int((created - epoch).total_seconds()))
        elif args.element_type == 'tag':
            filepath = create_stub_file('Tags', args.tag_name)

            # Add the title which should be the tag name
            with open(filepath, 'a') as f:
                # TODO: Would be very cool to reinforce camel case for the file
                # name and canonical for title.
                f.write("\n# {0}\n".format(args.tag_name))
        else:
            raise NotImplementedError("Unknown information type: {0}"
                    .format(args.element_type))

        if filepath:
            subprocess.call(["vim", filepath])
            print("Created: {0}, {1}".format(args.element_type, filepath))
        else:
            print("Failed to create new information element")
    elif args.action_name == 'show':

        tags_dir = "{0}/{1}".format(js.root_directory, 'Tags')
        logs_dir = "{0}/{1}".format(js.root_directory, 'LogEntries')

        def open_file(file_dir, file_name):
            file_name = file_name if ".md" in file_name \
                else "{0}.md".format(file_name)
            filepath = "{0}/{1}".format(file_dir, file_name)

            if not os.path.isfile(filepath):
                raise IOError("File does not exist! {0}".format(filepath))

            # Previews the markdown. This will require you to change the
            # mimeapps.list setting file in order to chose your markdown preview
            # tool.
            filepath_browser = "file://{0}".format(filepath)
            webbrowser.open(filepath_browser)

        def convert_file_to_json(src_file):
            """
            Form a json representation of a log file.
            """
            d = src_file.read().split('\n')

            def parse_metadata(line):
                """
                Looking for Author, Created, Version, Tags

                :param line: line for metadata e.g. "Author: John Doe"
                :type line: string

                :return: (key string, value string)
                """
                m = re.search('^(Author|Created|Version|Tags): (.*)', line)
                return m.group(1).lower(), m.group(2)

            t = (parse_metadata(d[i]) for i in range(0, 4))
            response = dict(t)
            response['tags'] = response['tags'].split(', ')
            response['body'] = "\n".join(d[5:])
            return response

        def convert_json_to_summary(src_json):
            """
            Form a summary representation of the log file.
            """
            # Clip the body
            clip = src_json['body'].split('\n')[0][0:250]
            return "\n".join([log_file, ", ".join(src_json['tags']), clip])

        if args.show_type == 'tags':
            tag_pattern = re.compile('(\w*)\.md')

            for tag_file in sorted(os.listdir(tags_dir)):
                tag = tag_pattern.search(tag_file).group(1)
                print(tag)
        elif args.show_type == 'tag':
            open_file(tags_dir, args.tag_name)
        elif args.show_type == 'logs':
            entries = []

            # 1. Go through each log file
            # 2. Convert each file to json
            # 3. Filter by tag if need be
            # 4. Print entries

            # NOTE: The sort order is increasing in time because the most recent
            # should be visible at the new command prompt.
            for log_file in sorted(os.listdir(logs_dir), reverse=False):
                log_path = "{0}/{1}".format(logs_dir, log_file)

                with open(log_path, 'r') as f:
                    json_rep = convert_file_to_json(f)

                if not args.tag or any([args.tag.lower() in tag.lower()
                    for tag in json_rep['tags']]):
                    entries.append(convert_json_to_summary(json_rep))

            if entries:
                print("\n\n".join(entries))
                print("\n\nLog entries found: {0}".format(len(entries)))
            else:
                print("No log entries found")
        elif args.show_type == 'lastlog':
            for log_file in sorted(os.listdir(logs_dir), reverse=True):
                log_path = "{0}/{1}".format(logs_dir, log_file)

                with open(log_path, 'r') as f:
                    json_rep = convert_file_to_json(f)

                if any([args.tag.lower() in tag.lower()
                    for tag in json_rep['tags']]):
                    open_file(logs_dir, log_file)
                    break

            print("There is no log entry for \"{0}\"".format(args.tag))
        elif args.show_type == 'log':
            open_file(logs_dir, args.log_entry_name)
        else:
            raise NotImplementedError("Unknown show type: {0}"
                    .format(args.show_type))
<commit_msg>Print no log found message only when that is true<commit_after>#!/usr/bin/env python

import os, subprocess, argparse, re
import webbrowser
from datetime import datetime

class JarvisSettings(object):

    def __init__(self):
        # TODO: Make a test mode where it writes to a test directory.
        self._env_dir_jarvis_root = os.environ['JARVIS_DIR_ROOT']
        self._env_author = os.environ['JARVIS_AUTHOR']
        self._env_version = "0.1.0"

    @property
    def root_directory(self):
        return self._env_dir_jarvis_root

    @property
    def author(self):
        return self._env_author

    @property
    def document_version(self):
        return self._env_version

if "__main__" == __name__:
    parser = argparse.ArgumentParser(description='Jarvis is used for personal information management')

    subparsers = parser.add_subparsers(help='Actions for Jarvis',
            dest='action_name')

    # New actions
    parser_new = subparsers.add_parser('new', help='Create an information element')
    subparsers_new = parser_new.add_subparsers(help='Types of new information element',
            dest='element_type')
    parser_new_log = subparsers_new.add_parser('log', help='Create a new log entry')
    parser_new_tag = subparsers_new.add_parser('tag', help='Create a new tag element')
    parser_new_tag.add_argument('tag_name', help='Tag name')

    # Show actions
    parser_show = subparsers.add_parser('show', help='Show information elements')
    subparsers_show = parser_show.add_subparsers(help='Types of show actions',
            dest='show_type')

    parser_show_logs = subparsers_show.add_parser('logs', help='List all logs')
    parser_show_logs.add_argument('-t', '--tag', nargs='?', help='Tag to search')

    parser_show_lastlog = subparsers_show.add_parser('lastlog',
            help='Open last log entry in the browser')
    parser_show_lastlog.add_argument('-t', '--tag', nargs='?', required=True,
            help='Tag to search')

    parser_show_log = subparsers_show.add_parser('log', help='Open log entry in the browser')
    parser_show_log.add_argument('log_entry_name', help='Log name')

    parser_show_tags = subparsers_show.add_parser('tags', help='List all tags')

    parser_show_tag = subparsers_show.add_parser('tag', help='Open tag in the browser')
    parser_show_tag.add_argument('tag_name', help='Tag name')

    args = parser.parse_args()

    # NOTE: Argparse should filter and validate and ensure that only the known
    # choices get through to this point.
    # UPDATE: There is a case where the subsubparser allows for Nones to slip
    # through. The required flag doesn't seem to be reinforced so the following
    # is acceptable to Argparse which sucks for me:
    #   jarvis.py new

    js = JarvisSettings()

    if args.action_name == 'new':
        created = datetime.utcnow().replace(microsecond=0)

        def create_stub_file(subdir_name, filename):
            dir_target = "{0}/{1}".format(js.root_directory, subdir_name)

            filepath = "{0}/{1}.md".format(dir_target, filename)

            if os.path.isfile(filepath):
                raise IOError("File already exists! {0}".format(filepath))

            with open(filepath, 'w') as f:
                f.write("Author: {0}\n".format(js.author))
                f.write("Created: {0}\n".format(created.isoformat()))
                f.write("Version: {0}\n".format(js.document_version))
                f.write("Tags: \n")

            return filepath

        if args.element_type == 'log':
            # datetime.fromtimestamp(0) is not Unix epoch and returns
            # 1969-12-31 19:00 instead.
            epoch = datetime(1970, 1, 1)

            filepath = create_stub_file('LogEntries',
                int((created - epoch).total_seconds()))
        elif args.element_type == 'tag':
            filepath = create_stub_file('Tags', args.tag_name)

            # Add the title which should be the tag name
            with open(filepath, 'a') as f:
                # TODO: Would be very cool to reinforce camel case for the file
                # name and canonical for title.
                f.write("\n# {0}\n".format(args.tag_name))
        else:
            raise NotImplementedError("Unknown information type: {0}"
                    .format(args.element_type))

        if filepath:
            subprocess.call(["vim", filepath])
            print("Created: {0}, {1}".format(args.element_type, filepath))
        else:
            print("Failed to create new information element")
    elif args.action_name == 'show':

        tags_dir = "{0}/{1}".format(js.root_directory, 'Tags')
        logs_dir = "{0}/{1}".format(js.root_directory, 'LogEntries')

        def open_file(file_dir, file_name):
            file_name = file_name if ".md" in file_name \
                else "{0}.md".format(file_name)
            filepath = "{0}/{1}".format(file_dir, file_name)

            if not os.path.isfile(filepath):
                raise IOError("File does not exist! {0}".format(filepath))

            # Previews the markdown. This will require you to change the
            # mimeapps.list setting file in order to chose your markdown preview
            # tool.
            filepath_browser = "file://{0}".format(filepath)
            webbrowser.open(filepath_browser)

        def convert_file_to_json(src_file):
            """
            Form a json representation of a log file.
            """
            d = src_file.read().split('\n')

            def parse_metadata(line):
                """
                Looking for Author, Created, Version, Tags

                :param line: line for metadata e.g. "Author: John Doe"
                :type line: string

                :return: (key string, value string)
                """
                m = re.search('^(Author|Created|Version|Tags): (.*)', line)
                return m.group(1).lower(), m.group(2)

            t = (parse_metadata(d[i]) for i in range(0, 4))
            response = dict(t)
            response['tags'] = response['tags'].split(', ')
            response['body'] = "\n".join(d[5:])
            return response

        def convert_json_to_summary(src_json):
            """
            Form a summary representation of the log file.
            """
            # Clip the body
            clip = src_json['body'].split('\n')[0][0:250]
            return "\n".join([log_file, ", ".join(src_json['tags']), clip])

        if args.show_type == 'tags':
            tag_pattern = re.compile('(\w*)\.md')

            for tag_file in sorted(os.listdir(tags_dir)):
                tag = tag_pattern.search(tag_file).group(1)
                print(tag)
        elif args.show_type == 'tag':
            open_file(tags_dir, args.tag_name)
        elif args.show_type == 'logs':
            entries = []

            # 1. Go through each log file
            # 2. Convert each file to json
            # 3. Filter by tag if need be
            # 4. Print entries

            # NOTE: The sort order is increasing in time because the most recent
            # should be visible at the new command prompt.
            for log_file in sorted(os.listdir(logs_dir), reverse=False):
                log_path = "{0}/{1}".format(logs_dir, log_file)

                with open(log_path, 'r') as f:
                    json_rep = convert_file_to_json(f)

                if not args.tag or any([args.tag.lower() in tag.lower()
                    for tag in json_rep['tags']]):
                    entries.append(convert_json_to_summary(json_rep))

            if entries:
                print("\n\n".join(entries))
                print("\n\nLog entries found: {0}".format(len(entries)))
            else:
                print("No log entries found")
        elif args.show_type == 'lastlog':
            is_found = False

            for log_file in sorted(os.listdir(logs_dir), reverse=True):
                log_path = "{0}/{1}".format(logs_dir, log_file)

                with open(log_path, 'r') as f:
                    json_rep = convert_file_to_json(f)

                if any([args.tag.lower() in tag.lower()
                    for tag in json_rep['tags']]):
                    open_file(logs_dir, log_file)
                    is_found = True
                    break

            if not is_found:
                print("There is no log entry for \"{0}\"".format(args.tag))
        elif args.show_type == 'log':
            open_file(logs_dir, args.log_entry_name)
        else:
            raise NotImplementedError("Unknown show type: {0}"
                    .format(args.show_type))
<|endoftext|>


<commit_before># Copyright (c) Citrix Systems Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms,
# with or without modification, are permitted provided
# that the following conditions are met:
#
# *   Redistributions of source code must retain the above
#     copyright notice, this list of conditions and the
#     following disclaimer.
# *   Redistributions in binary form must reproduce the above
#     copyright notice, this list of conditions and the
#     following disclaimer in the documentation and/or other
#     materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.

artifactory='https://repo.citrite.net:443/xs-local-build/'

build_tar_source_files = {
        "xenguestagent" : r'win-xenguestagent/master/win-xenguestagent-219/xenguestagent.tar',
        "xenbus": r"win-xenbus/patchq-8.2/win-xenbus-113/xenbus.inspur.signed.tar",
        "xenvif": r"win-xenvif/patchq/win-xenvif-152/xenvif.inspur.signed.tar",
        "xennet": r"win-xennet/patchq/win-xennet-64/xennet.inspur.signed.tar",
        "xeniface": r"win-xeniface/patchq/win-xeniface-103/xeniface.inspur.signed.tar",
        "xenvbd": r"win-xenvbd/patchq/win-xenvbd-158/xenvbd.inspur.signed.tar",
        "xenvss" : r'win-xenvss/master/win-xenvss-18/xenvss.tar',
}

signed_drivers = {
}

all_drivers_signed = True
<commit_msg>XIface 103 update<commit_after># Copyright (c) Citrix Systems Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms,
# with or without modification, are permitted provided
# that the following conditions are met:
#
# *   Redistributions of source code must retain the above
#     copyright notice, this list of conditions and the
#     following disclaimer.
# *   Redistributions in binary form must reproduce the above
#     copyright notice, this list of conditions and the
#     following disclaimer in the documentation and/or other
#     materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.

artifactory='https://repo.citrite.net:443/xs-local-build/'

build_tar_source_files = {
        "xenguestagent" : r'win-xenguestagent/master/win-xenguestagent-219/xenguestagent.tar',
        "xenbus": r"win-xenbus/patchq-8.2/win-xenbus-113/xenbus.inspur.signed.tar",
        "xenvif": r"win-xenvif/patchq/win-xenvif-152/xenvif.inspur.signed.tar",
        "xennet": r"win-xennet/patchq/win-xennet-64/xennet.inspur.signed.tar",
        "xeniface": r"win-xeniface/patchq/win-xeniface-103/xeniface.inspur.tar",
        "xenvbd": r"win-xenvbd/patchq/win-xenvbd-158/xenvbd.inspur.signed.tar",
        "xenvss" : r'win-xenvss/master/win-xenvss-18/xenvss.tar',
}

signed_drivers = {
}

all_drivers_signed = False
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

###############################################################################
# Information
###############################################################################
# Created by Linwood Creekmore
# Starter file by Danny Holloway of HumanGeo
# Significant input from Vikram Mittal

# In partial fulfillment of the requirements for the Georgetown University Data Analytics Graduate Certificate Program

# March 5, 2015

# https://plus.google.com/+LinwoodCreekmoreIII/

###############################################################################
# Imports
###############################################################################

import math
import unicodecsv
import pandas as pd
import matplotlib.pyplot as plt
from unum.units import *
import numpy as np
from pylab import plot
import os
import math
import statistics
from os import walk
from os.path import basename
import unicodecsv as csv
import logging as log
import time

###############################################################################
# File Paths
###############################################################################

path        = os.path.abspath(os.getcwd())
LOG_DIR     = os.path.normpath(os.path.join(os.path.dirname(path),os.path.basename(path),"log"))
INPUT_DIR   = os.path.normpath(os.path.join(os.path.dirname(path),os.path.basename(path),"output","test"))
OUTPUT_DIR  = os.path.normpath(os.path.join(os.path.dirname(path),os.path.basename(path),"output","trip"))
OUTPUT_DIR2 = os.path.normpath(os.path.join(os.path.dirname(path),os.path.basename(path),"output","aggregate"))

#print "Home: %s, \nAbsolute: %s \nInput:%s \nOutput: %s." % (
    #path, INPUT_DIR, OUTPUT_DIR2, OUTPUT_DIR)

###############################################################################
# Create Logger, File Handlers, Formatters, and Filters (as needed)
###############################################################################

class SingleLevelFilter(log.Filter):
    def __init__(self, passlevel, reject):
        self.passlevel = passlevel
        self.reject = reject

    def filter(self, record):
        if self.reject:
            return (record.levelno != self.passlevel)
        else:
            return (record.levelno == self.passlevel)

logger = log.getLogger("DataQuality")
logger.setLevel(log.DEBUG)

infoHandler = log.FileHandler(os.path.join(LOG_DIR,"Displacement_Summary.log"), mode='w')
infoFilter = SingleLevelFilter(log.INFO, False)
infoHandler.addFilter(infoFilter)

warnHandler = log.FileHandler(os.path.join(LOG_DIR,"Warnings.log"), mode='w')
warnFilter = SingleLevelFilter(log.WARNING, False)
warnHandler.addFilter(warnFilter)

logger.addHandler(infoHandler)
logger.addHandler(warnHandler)

###############################################################################
# Helper Functions
###############################################################################

# This is a generic dot product or Euclidean distance function

def dotproduct(x,y):
    return math.sqrt(x**2 + y**2)

# This is used to convert the meters per second velocity into a miles per hour value.
def tomph(velocity):
    return velocity * 2.24

'''
This function is used to convert the units for increment traveled from meters to feet; because we measure over each second,
 a more human readable unit is needed.
'''
def meterstofeet(args):
    return args * 3.28

def getDriver(dirName):
    return (basename(dirName))

def getFileBaseName(baseName):
    return fileName.split('.')[0]

def getTrip(baseName):
    return int(getFileBaseName(baseName))

def isFloat(str):
    try:
        float(str)
        return True
    except:
        return False

###############################################################################
# Key Metric Functions
###############################################################################

'''
We are dealing with a positon vector; therefore, we must calculate the x and y components of the position vector (i.e. car's position at point of time) for the car as it moves in space.
Velocity is measured by dividing the change in position by the change in time.
The magnitude of the velocity vector can be found using the scalar dot product, which is equivalent to using the Pythagorean theorem.
Acceleration is measured by dividing the change in velocity by the change in time
'''

# Calculate total Displacement; since we always start from 0,0, it's just the dot product of the last point

# Calculate the average horizontal and vertical component of velocity; we ignore time because it's always equal to 1 second
def getVelocity(x,y,last_x,last_y):
    x_avg_vel = abs(abs(x) - abs(last_x))
    y_avg_vel = abs(abs(y) - abs(last_y))
    return x_avg_vel, y_avg_vel

# Calculate the horizontal and vertical acceleration components; this will be critical to determining turns, braking, etc.
def getAcceleration(x_avg_vel,last_x_avg_vel,y_avg_vel,last_y_avg_vel):
    return x_avg_acl,y_avg_acl

# Calculates the NEW heading of the vehicle
def heading(y,x,last_y,last_x):
    return getDirection(y-last_y,x-last_x)

# Calculate the direction
def getDirection(y,x):
    direction_rad = math.atan2(y,x)
    direction = math.degrees(direction_rad)
    if direction < 0:
        direction += 360
    return direction

# Calculate displacement
def getIncrement(x,last_x,y,last_y):
    increment_traveled = math.sqrt( (x - last_x)**2 + (y - last_y)**2 )
    return increment_traveled

###
#This calculation splits cardinal directions into buffer zones of 45 degrees each. We get the directional heading from
#the x,y location and output a string based on the buffer zone conditional statement below
###

def getCardinalDirection(direction):
    carddir = ''
    if direction >= 0 and direction <= 22.5:
        carddir = 'East'
    elif direction > 337.5 and direction <=360:
        carddir = 'East'
    elif direction > 22.5 and direction <= 67.5:
        carddir = 'Northeast'
    elif direction > 67.5 and direction <= 112.5:
        carddir = 'North'
    elif direction > 112.5 and direction <= 157.5:
        carddir  = 'Northwest'
    elif direction > 157.5 and direction <= 202.5:
        carddir  = 'West'
    elif direction > 202.5 and direction <= 247.5:
        carddir = 'Southwest'
    elif direction > 247.5 and direction <= 292.5:
        carddir  = 'South'
    elif direction > 292.5 and direction <= 337.5:
        carddir  = 'Southeast'

    return carddir

###############################################################################
# Main Functions
###############################################################################

def createFile(dirName, fileName):
    driver = getDriver(dirName)         # obtain driver id from directory name
    #df2 = pd.read_csv(os.path.join(OUTPUT_DIR,fileName))

    fileName.split('_')

    # This will open the driver directory/folder in a directory with all the driver files.  This should go through all 200 trip files for one driver.

    with open(os.path.join(os.path.normpath(path), dirName, fileName), 'rU') as infile:
        reader = csv.DictReader(infile, delimiter=',', quotechar='"')

        # This creates the output csv file that will hold all the calculated metrics

        with open(os.path.join(OUTPUT_DIR,fileName), 'wb') as outfile:
            writer = csv.writer(outfile)

            # This writes the header row for our output file using trip/driver IDs from Vik's IDify.py file

            for idx, row in enumerate(reader):
                if idx == 0:
                    trip_id = 1

                    writer.writerow(['driver_id', 'trip_id', 'Velocity (mph)', 'Acceleration (mph per s)','Time (s)', 'Increment Traveled (feet)','Change in Direction per s','Direction (deg)','Direction(card)'])

                    if not row == { 'driver_id':'driver_id', 'trip_id':'trip_id', 'x':'x', 'y':'y'}:

                        logger.warning("Missing or invalid header for driver %s and trip %s" % ("driver", trip_id))

                    else:

                        if len(row) != 4:

                            logger.warning("Too few/many values in row %s for driver %s and trip %s" % (idx, driver, trip_id))

                    # skip first line with headings.

                    infile.next()

                    # Here we establish all the zero values for the key metrics we iterate.  These are defined in the key metrics documentation.

                    last_x, last_y, = 0.0, 0.0
                    seconds = 0
                    distance = 0
                    stops = 0
                    braking_event = 0
                    last_x_avg_vel= 0
                    last_y_avg_vel= 0
                    max_velocity = 0
                    accelerations = 0
                    decelerations = 0
                    x_avg_vel = 0
                    y_avg_vel = 0
                    x_avg_acl = 0
                    y_avg_acl = 0
                    last_heading = 0.0
                    cur_heading = 0.0
                    lastvel = 0
                    lastaccel = 0

                    # Creating an list to append all the calculated key metric values
                    metrics = []

                    # Establish the trip id value to iterate over for our database
                    #trip_id = 1

                    # We loop through the row values of x,y and calculate the key metric values, and then append the value to the metrics list above.

                    for l in infile:

                        driver_id, trip_id, x, y, = l.split(',')
                        driver_id, trip_id, x, y = int(driver_id), int(trip_id), float(x), float(y)
                        x_avg_vel,y_avg_vel = getVelocity(x,y,last_x,last_y)

                        metrics.append(fileName.split('_')[0]) #append driver #
                        metrics.append(trip_id) #append trip id
                        velocity = tomph(dotproduct(x_avg_vel, y_avg_vel))
                        if velocity > 200:
                            velocity = lastvel

                        metrics.append(velocity)

                        holdingvel = velocity - lastvel
                        if abs(holdingvel) > 200:
                            holdingvel = lastaccel

                        metrics.append(holdingvel) #acceleration
                        metrics.append(seconds) #time
                        metrics.append(getIncrement(x,last_x,y,last_y)) #distance traveled

                        last_heading = cur_heading
                        cur_heading = heading(y,x, last_y, last_x)
                        holding = abs(cur_heading - last_heading)
                        if holding >= 45 and getIncrement(x,last_x,y,last_y) <= 2:
                            holding = 0
                        elif holding >= 180:
                            holding = 360 - holding

                        metrics.append(holding) # enters change in direction
                        metrics.append(heading(y,x, last_y, last_x)) # calculates the heading in degrees
                        metrics.append(getCardinalDirection(heading(y,x,last_y,last_x))) # calculates cardinal position

                        # We write the identifying and key metrics values to our csv

                        writer.writerow(metrics)

                        # Next, we clear our metrics list for the next loop, iterate our time and trip ID values, and store the previous locations and velocity.

                        metrics = []
                        seconds += 1
                        trip_id += 1
                        last_x, last_y = x, y
                        lastvel = velocity
                        holdingvel = lastaccel
                        last_x_avg_vel, last_y_avg_vel = x_avg_vel, y_avg_vel
                        direction = last_heading

    with open(os.path.join(OUTPUT_DIR, fileName), 'rU') as infile:
        reader = csv.DictReader(infile, delimiter=',', quotechar='"')

        with open(os.path.join(OUTPUT_DIR2,fileName), 'wb') as outfile:
            writer = csv.writer(outfile, lineterminator = '\n')

            df = pd.read_csv(os.path.join(OUTPUT_DIR,fileName))
            writer.writerow(['driver_id', 'trip_id', 'Average Velocity (mph)', 'Max Velocity', 'Velocity Stdev','Average Acceleration (mph per s)', 'Max Acceleration (mph per s)', ' Acceleration Stdev','Displacement (meters)''Max Direction Change per sec', ' Direction Stdev','Time (s)'])

            agvalues = []

            agvalues.append(fileName.split('_')[0])
            agvalues.append(df.loc[1][1])
            agvalues.append(df.loc[2:]['Velocity (mph)'].mean())
            agvalues.append(df.loc[2:]['Velocity (mph)'].max())
            agvalues.append(statistics.stdev(df.loc[1:]['Velocity (mph)']))
            agvalues.append(df.loc[2:]['Acceleration (mph per s)'].mean())
            agvalues.append(df.loc[2:]['Acceleration (mph per s)'].max())
            agvalues.append(statistics.stdev(df.loc[1:]['Acceleration (mph per s)']))
            #agvalues.append(dotproduct(df2.loc[(len(df2)-1)]['x'],df2.loc[(len(df2)-1)]['y']))
            agvalues.append(df.loc[2:]['Change in Direction per s'].max())
            agvalues.append(statistics.stdev(df.loc[2:]['Direction (deg)']))
            agvalues.append(df.loc[1:]['Time (s)'].max())

            writer.writerow(agvalues)

            agvalues = []

###############################################################################
# 'Main' Function
###############################################################################

if __name__ == '__main__':
    for dirName, subdirList, fileList in walk(INPUT_DIR):
        for fileName in fileList:
            if fileName.endswith('.csv'):
                '''
                for dirName, subdirList, fileList in walk(INPUT_DIR):
                    if basename(dirName).isdigit():                         # directory likely represents a driver
                        for fileName in fileList:
                            if fileName.endswith('.csv') :       # trip data is in .csv files and a numeric file name is likely a trip file, and getFileBaseName(fileName).isdigit()
                '''
                createFile(dirName, fileName)
    os.system('python lin.py')<commit_msg>Updated files<commit_after># -*- coding: utf-8 -*-

###############################################################################
# Information
###############################################################################
# Created by Linwood Creekmore
# Starter file by Danny Holloway of HumanGeo
# Significant input from Vikram Mittal

# In partial fulfillment of the requirements for the Georgetown University Data Analytics Graduate Certificate Program

# March 5, 2015

# https://plus.google.com/+LinwoodCreekmoreIII/

###############################################################################
# Imports
###############################################################################

import math
import unicodecsv
import pandas as pd
import matplotlib.pyplot as plt
from unum.units import *
import numpy as np
from pylab import plot
import os
import math
import statistics
from os import walk
from os.path import basename
import unicodecsv as csv
import logging as log
import time

###############################################################################
# File Paths
###############################################################################

path        = os.path.abspath(os.getcwd())
LOG_DIR     = os.path.normpath(os.path.join(os.path.dirname(path),os.path.basename(path),"log"))
INPUT_DIR   = os.path.normpath(os.path.join(os.path.dirname(path),os.path.basename(path),"output","test"))
OUTPUT_DIR  = os.path.normpath(os.path.join(os.path.dirname(path),os.path.basename(path),"output","trip"))
OUTPUT_DIR2 = os.path.normpath(os.path.join(os.path.dirname(path),os.path.basename(path),"output","aggregate"))

#print "Home: %s, \nAbsolute: %s \nInput:%s \nOutput: %s." % (
    #path, INPUT_DIR, OUTPUT_DIR2, OUTPUT_DIR)

###############################################################################
# Create Logger, File Handlers, Formatters, and Filters (as needed)
###############################################################################

class SingleLevelFilter(log.Filter):
    def __init__(self, passlevel, reject):
        self.passlevel = passlevel
        self.reject = reject

    def filter(self, record):
        if self.reject:
            return (record.levelno != self.passlevel)
        else:
            return (record.levelno == self.passlevel)

logger = log.getLogger("DataQuality")
logger.setLevel(log.DEBUG)

infoHandler = log.FileHandler(os.path.join(LOG_DIR,"Displacement_Summary.log"), mode='w')
infoFilter = SingleLevelFilter(log.INFO, False)
infoHandler.addFilter(infoFilter)

warnHandler = log.FileHandler(os.path.join(LOG_DIR,"Warnings.log"), mode='w')
warnFilter = SingleLevelFilter(log.WARNING, False)
warnHandler.addFilter(warnFilter)

logger.addHandler(infoHandler)
logger.addHandler(warnHandler)

###############################################################################
# Helper Functions
###############################################################################

# This is a generic dot product or Euclidean distance function

def dotproduct(x,y):
    return math.sqrt(x**2 + y**2)

# This is used to convert the meters per second velocity into a miles per hour value.
def tomph(velocity):
    return velocity * 2.24

'''
This function is used to convert the units for increment traveled from meters to feet; because we measure over each second,
 a more human readable unit is needed.
'''
def meterstofeet(args):
    return args * 3.28

def getDriver(dirName):
    return (basename(dirName))

def getFileBaseName(baseName):
    return fileName.split('.')[0]

def getTrip(baseName):
    return int(getFileBaseName(baseName))

def isFloat(str):
    try:
        float(str)
        return True
    except:
        return False

###############################################################################
# Key Metric Functions
###############################################################################

'''
We are dealing with a positon vector; therefore, we must calculate the x and y components of the position vector (i.e. car's position at point of time) for the car as it moves in space.
Velocity is measured by dividing the change in position by the change in time.
The magnitude of the velocity vector can be found using the scalar dot product, which is equivalent to using the Pythagorean theorem.
Acceleration is measured by dividing the change in velocity by the change in time
'''

# Calculate total Displacement; since we always start from 0,0, it's just the dot product of the last point

# Calculate the average horizontal and vertical component of velocity; we ignore time because it's always equal to 1 second
def getVelocity(x,y,last_x,last_y):
    x_avg_vel = abs(abs(x) - abs(last_x))
    y_avg_vel = abs(abs(y) - abs(last_y))
    return x_avg_vel, y_avg_vel

# Calculate the horizontal and vertical acceleration components; this will be critical to determining turns, braking, etc.
def getAcceleration(x_avg_vel,last_x_avg_vel,y_avg_vel,last_y_avg_vel):
    return x_avg_acl,y_avg_acl

# Calculates the NEW heading of the vehicle
def heading(y,x,last_y,last_x):
    return getDirection(y-last_y,x-last_x)

# Calculate the direction
def getDirection(y,x):
    direction_rad = math.atan2(y,x)
    direction = math.degrees(direction_rad)
    if direction < 0:
        direction += 360
    return direction

# Calculate displacement
def getIncrement(x,last_x,y,last_y):
    increment_traveled = math.sqrt( (x - last_x)**2 + (y - last_y)**2 )
    return increment_traveled

###
#This calculation splits cardinal directions into buffer zones of 45 degrees each. We get the directional heading from
#the x,y location and output a string based on the buffer zone conditional statement below
###

def getCardinalDirection(direction):
    carddir = ''
    if direction >= 0 and direction <= 22.5:
        carddir = 'East'
    elif direction > 337.5 and direction <=360:
        carddir = 'East'
    elif direction > 22.5 and direction <= 67.5:
        carddir = 'Northeast'
    elif direction > 67.5 and direction <= 112.5:
        carddir = 'North'
    elif direction > 112.5 and direction <= 157.5:
        carddir  = 'Northwest'
    elif direction > 157.5 and direction <= 202.5:
        carddir  = 'West'
    elif direction > 202.5 and direction <= 247.5:
        carddir = 'Southwest'
    elif direction > 247.5 and direction <= 292.5:
        carddir  = 'South'
    elif direction > 292.5 and direction <= 337.5:
        carddir  = 'Southeast'

    return carddir

###############################################################################
# Main Functions
###############################################################################

def createFile(dirName, fileName):
    driver = getDriver(dirName)         # obtain driver id from directory name
    #df2 = pd.read_csv(os.path.join(OUTPUT_DIR,fileName))

    fileName.split('_')

    # This will open the driver directory/folder in a directory with all the driver files.  This should go through all 200 trip files for one driver.

    with open(os.path.join(os.path.normpath(path), dirName, fileName), 'rU') as infile:
        reader = csv.DictReader(infile, delimiter=',', quotechar='"')

        # This creates the output csv file that will hold all the calculated metrics

        with open(os.path.join(OUTPUT_DIR,fileName), 'wb') as outfile:
            writer = csv.writer(outfile)

            # This writes the header row for our output file using trip/driver IDs from Vik's IDify.py file

            for idx, row in enumerate(reader):
                if idx == 0:
                    trip_id = 1

                    writer.writerow(['driver_id', 'trip_id', 'Velocity (mph)', 'Acceleration (mph per s)','Time (s)', 'Increment Traveled (feet)','Change in Direction per s','Direction (deg)','Direction(card)'])

                    if not row == { 'driver_id':'driver_id', 'trip_id':'trip_id', 'x':'x', 'y':'y'}:

                        logger.warning("Missing or invalid header for driver %s and trip %s" % ("driver", trip_id))

                    else:

                        if len(row) != 4:

                            logger.warning("Too few/many values in row %s for driver %s and trip %s" % (idx, driver, trip_id))

                    # skip first line with headings.

                    infile.next()

                    # Here we establish all the zero values for the key metrics we iterate.  These are defined in the key metrics documentation.

                    last_x, last_y, = 0.0, 0.0
                    seconds = 0
                    distance = 0
                    stops = 0
                    braking_event = 0
                    last_x_avg_vel= 0
                    last_y_avg_vel= 0
                    max_velocity = 0
                    accelerations = 0
                    decelerations = 0
                    x_avg_vel = 0
                    y_avg_vel = 0
                    x_avg_acl = 0
                    y_avg_acl = 0
                    last_heading = 0.0
                    cur_heading = 0.0
                    lastvel = 0
                    lastaccel = 0

                    # Creating an list to append all the calculated key metric values
                    metrics = []

                    # Establish the trip id value to iterate over for our database
                    #trip_id = 1

                    # We loop through the row values of x,y and calculate the key metric values, and then append the value to the metrics list above.

                    for l in infile:

                        driver_id, trip_id, x, y, = l.split(',')
                        driver_id, trip_id, x, y = int(driver_id), int(trip_id), float(x), float(y)
                        x_avg_vel,y_avg_vel = getVelocity(x,y,last_x,last_y)

                        metrics.append(fileName.split('_')[0]) #append driver #
                        metrics.append(trip_id) #append trip id
                        velocity = tomph(dotproduct(x_avg_vel, y_avg_vel))
                        if velocity > 200:
                            velocity = lastvel

                        metrics.append(velocity)

                        holdingvel = velocity - lastvel
                        if abs(holdingvel) > 200:
                            holdingvel = lastaccel

                        metrics.append(holdingvel) #acceleration
                        metrics.append(seconds) #time
                        metrics.append(getIncrement(x,last_x,y,last_y)) #distance traveled

                        last_heading = cur_heading
                        cur_heading = heading(y,x, last_y, last_x)
                        holding = abs(cur_heading - last_heading)
                        if holding >= 45 and getIncrement(x,last_x,y,last_y) <= 2:
                            holding = 0
                        elif holding >= 180:
                            holding = 360 - holding

                        metrics.append(holding) # enters change in direction
                        metrics.append(heading(y,x, last_y, last_x)) # calculates the heading in degrees
                        metrics.append(getCardinalDirection(heading(y,x,last_y,last_x))) # calculates cardinal position

                        # We write the identifying and key metrics values to our csv

                        writer.writerow(metrics)

                        # Next, we clear our metrics list for the next loop, iterate our time and trip ID values, and store the previous locations and velocity.

                        metrics = []
                        seconds += 1
                        trip_id += 1
                        last_x, last_y = x, y
                        lastvel = velocity
                        holdingvel = lastaccel
                        last_x_avg_vel, last_y_avg_vel = x_avg_vel, y_avg_vel
                        direction = last_heading

    with open(os.path.join(OUTPUT_DIR, fileName), 'rU') as infile:
        reader = csv.DictReader(infile, delimiter=',', quotechar='"')

        with open(os.path.join(OUTPUT_DIR2,fileName), 'wb') as outfile:
            writer = csv.writer(outfile, lineterminator = '\n')

            df = pd.read_csv(os.path.join(OUTPUT_DIR,fileName))
            writer.writerow(['driver_id', 'trip_id', 'Average Velocity (mph)', 'Max Velocity', 'Velocity Stdev','Average Acceleration (mph per s)', 'Max Acceleration (mph per s)', ' Acceleration Stdev','Displacement (meters)','Max Direction Change per sec', ' Direction Stdev','Time (s)'])
            df2 = pd.read_csv(os.path.join(INPUT_DIR,str(driver),fileName))

            agvalues = []

            agvalues.append(fileName.split('_')[0])
            agvalues.append(df.loc[1][1])
            agvalues.append(df.loc[2:]['Velocity (mph)'].mean())
            agvalues.append(df.loc[2:]['Velocity (mph)'].max())
            agvalues.append(statistics.stdev(df.loc[1:]['Velocity (mph)']))
            agvalues.append(df.loc[2:]['Acceleration (mph per s)'].mean())
            agvalues.append(df.loc[2:]['Acceleration (mph per s)'].max())
            agvalues.append(statistics.stdev(df.loc[1:]['Acceleration (mph per s)']))
            displace = dotproduct(df2.loc[(len(df2)-1)]['x'],df2.loc[(len(df2)-1)]['y'])
            agvalues.append(displace)
            agvalues.append(df.loc[2:]['Change in Direction per s'].max())
            agvalues.append(statistics.stdev(df.loc[2:]['Direction (deg)']))
            agvalues.append(df.loc[1:]['Time (s)'].max())

            writer.writerow(agvalues)

            agvalues = []

###############################################################################
# 'Main' Function
###############################################################################

if __name__ == '__main__':
    for dirName, subdirList, fileList in walk(INPUT_DIR):
        for fileName in fileList:
            if fileName.endswith('.csv'):
                '''
                for dirName, subdirList, fileList in walk(INPUT_DIR):
                    if basename(dirName).isdigit():                         # directory likely represents a driver
                        for fileName in fileList:
                            if fileName.endswith('.csv') :       # trip data is in .csv files and a numeric file name is likely a trip file, and getFileBaseName(fileName).isdigit()
                '''
                createFile(dirName, fileName)
    os.system('python lin.py')<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
# Copyright 2017-2018 Rumma & Ko Ltd
# License: BSD (see file COPYING for details)

# $ python setup.py test -s tests.PackagesTests.test_packages

SETUP_INFO = dict(
    name='lino-tera',
    version='19.2.0',
    install_requires=['lino-xl'],
    # tests_require=['pytest', 'mock'],
    test_suite='tests',
    description=("A Lino application for managing therapeutic centres"),
    long_description="""\
.. image:: https://readthedocs.org/projects/lino/badge/?version=latest
    :alt: Documentation Status
    :target: http://lino.readthedocs.io/en/latest/?badge=latest

.. image:: https://coveralls.io/repos/github/lino-framework/tera/badge.svg?branch=master
    :target: https://coveralls.io/github/lino-framework/tera?branch=master

.. image:: https://travis-ci.org/lino-framework/tera.svg?branch=stable
    :target: https://travis-ci.org/lino-framework/tera?branch=stable

.. image:: https://img.shields.io/pypi/v/lino-tera.svg
    :target: https://pypi.python.org/pypi/lino-tera/

.. image:: https://img.shields.io/pypi/l/lino-tera.svg
    :target: https://pypi.python.org/pypi/lino-tera/

Lino Tera is a customizable management system for therapeutic centres.

- The central project homepage is http://tera.lino-framework.org

- Functional specification see
  http://www.lino-framework.org/specs/tera

- There is also a User's Manual in German: `online using Sphinx
  <http://de.tera.lino-framework.org/>`__ (work in progress) and `PDF
  using LibreOffice
  <https://github.com/lino-framework/tera/raw/master/docs/dl/Handbuch_Lino_Tera.pdf>`__
  (not maintained).

- For *introductions* and *commercial information* about Lino Tera
  please see `www.saffre-rumma.net
  <http://www.saffre-rumma.net/tera/>`__.

""",
    author='Luc Saffre',
    author_email='luc@lino-framework.org',
    url="http://tera.lino-framework.org",
    license='BSD License',
    classifiers="""\
Programming Language :: Python
Programming Language :: Python :: 2
Development Status :: 4 - Beta
Environment :: Web Environment
Framework :: Django
Intended Audience :: Developers
Intended Audience :: System Administrators
Intended Audience :: Information Technology
Intended Audience :: Customer Service
License :: OSI Approved :: BSD License
Operating System :: OS Independent
""".splitlines())

SETUP_INFO.update(packages=[str(n) for n in """
lino_tera
lino_tera.lib
lino_tera.lib.contacts
lino_tera.lib.contacts.fixtures
lino_tera.lib.cal
lino_tera.lib.cal.fixtures
lino_tera.lib.coachings
lino_tera.lib.coachings.fixtures
lino_tera.lib.courses
lino_tera.lib.courses.fixtures
lino_tera.lib.households
lino_tera.lib.households.fixtures
lino_tera.lib.lists
lino_tera.lib.lists.fixtures
lino_tera.lib.notes
lino_tera.lib.notes.fixtures
lino_tera.lib.sales
lino_tera.lib.sales.fixtures
lino_tera.lib.products
lino_tera.lib.teams
lino_tera.lib.teams.fixtures
lino_tera.lib.tera
lino_tera.lib.tera.fixtures
lino_tera.lib.users
lino_tera.lib.users.fixtures
lino_tera.lib.invoicing
lino_tera.lib.invoicing.fixtures
""".splitlines() if n])

SETUP_INFO.update(message_extractors={
    'lino_tera': [
        ('**/cache/**',          'ignore', None),
        ('**.py',                'python', None),
        ('**.js',                'javascript', None),
        ('**/config/**.html', 'jinja2', None),
    ],
})

SETUP_INFO.update(include_package_data=True, zip_safe=False)
# SETUP_INFO.update(package_data=dict())

# def add_package_data(package, *patterns):
#     l = SETUP_INFO['package_data'].setdefault(package, [])
#     l.extend(patterns)
#     return l

# l = add_package_data('lino_noi.lib.noi')
# for lng in 'de fr'.split():
#     l.append('locale/%s/LC_MESSAGES/*.mo' % lng)
<commit_msg>http://luc.lino-framework.org/blog/2019/0701.html<commit_after># -*- coding: UTF-8 -*-
# Copyright 2017-2018 Rumma & Ko Ltd
# License: BSD (see file COPYING for details)

# $ python setup.py test -s tests.PackagesTests.test_packages

SETUP_INFO = dict(
    name='lino-tera',
    version='19.2.0',
    install_requires=['lino-xl'],
    # tests_require=['pytest', 'mock'],
    test_suite='tests',
    description=("A Lino application for managing therapeutic centres"),
    long_description="""\
.. image:: https://readthedocs.org/projects/lino/badge/?version=latest
    :alt: Documentation Status
    :target: http://lino.readthedocs.io/en/latest/?badge=latest

.. image:: https://coveralls.io/repos/github/lino-framework/tera/badge.svg?branch=master
    :target: https://coveralls.io/github/lino-framework/tera?branch=master

.. image:: https://travis-ci.org/lino-framework/tera.svg?branch=stable
    :target: https://travis-ci.org/lino-framework/tera?branch=stable

.. image:: https://img.shields.io/pypi/v/lino-tera.svg
    :target: https://pypi.python.org/pypi/lino-tera/

.. image:: https://img.shields.io/pypi/l/lino-tera.svg
    :target: https://pypi.python.org/pypi/lino-tera/

Lino Tera is a customizable management system for therapeutic centres.

- The central project homepage is http://tera.lino-framework.org

- Functional specification see
  http://www.lino-framework.org/specs/tera

- There is also a User's Manual in German: `online using Sphinx
  <http://de.tera.lino-framework.org/>`__ (work in progress) and `PDF
  using LibreOffice
  <https://github.com/lino-framework/tera/raw/master/docs/dl/Handbuch_Lino_Tera.pdf>`__
  (not maintained).

- For *introductions* and *commercial information* about Lino Tera
  please see `www.saffre-rumma.net
  <http://www.saffre-rumma.net/tera/>`__.

""",
    author='Luc Saffre',
    author_email='luc@lino-framework.org',
    url="http://tera.lino-framework.org",
    license='BSD-2-Clause',
    classifiers="""\
Programming Language :: Python
Programming Language :: Python :: 2
Development Status :: 4 - Beta
Environment :: Web Environment
Framework :: Django
Intended Audience :: Developers
Intended Audience :: System Administrators
Intended Audience :: Information Technology
Intended Audience :: Customer Service
License :: OSI Approved :: BSD License
Operating System :: OS Independent
""".splitlines())

SETUP_INFO.update(packages=[str(n) for n in """
lino_tera
lino_tera.lib
lino_tera.lib.contacts
lino_tera.lib.contacts.fixtures
lino_tera.lib.cal
lino_tera.lib.cal.fixtures
lino_tera.lib.coachings
lino_tera.lib.coachings.fixtures
lino_tera.lib.courses
lino_tera.lib.courses.fixtures
lino_tera.lib.households
lino_tera.lib.households.fixtures
lino_tera.lib.lists
lino_tera.lib.lists.fixtures
lino_tera.lib.notes
lino_tera.lib.notes.fixtures
lino_tera.lib.sales
lino_tera.lib.sales.fixtures
lino_tera.lib.products
lino_tera.lib.teams
lino_tera.lib.teams.fixtures
lino_tera.lib.tera
lino_tera.lib.tera.fixtures
lino_tera.lib.users
lino_tera.lib.users.fixtures
lino_tera.lib.invoicing
lino_tera.lib.invoicing.fixtures
""".splitlines() if n])

SETUP_INFO.update(message_extractors={
    'lino_tera': [
        ('**/cache/**',          'ignore', None),
        ('**.py',                'python', None),
        ('**.js',                'javascript', None),
        ('**/config/**.html', 'jinja2', None),
    ],
})

SETUP_INFO.update(include_package_data=True, zip_safe=False)
# SETUP_INFO.update(package_data=dict())

# def add_package_data(package, *patterns):
#     l = SETUP_INFO['package_data'].setdefault(package, [])
#     l.extend(patterns)
#     return l

# l = add_package_data('lino_noi.lib.noi')
# for lng in 'de fr'.split():
#     l.append('locale/%s/LC_MESSAGES/*.mo' % lng)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from os import sys, path
import json
import riak
import gc
from datetime import datetime

from pandas import DataFrame
from sqlalchemy import create_engine
from sqlalchemy.sql import text

sys.path.append(path.dirname(path.dirname(path.abspath(__file__))))
from utils import fix_render, conf, log_it

from bottle.ext.mongo import MongoPlugin

def run(cube_slug=None):
    mongo = MongoPlugin(
        uri=conf("mongodb")["uri"],
        db=conf("mongodb")["db"],
        json_mongo=True).get_mongo()

    MyClient = riak.RiakClient(
        protocol=conf("riak")["protocol"],
        http_port=conf("riak")["http_port"],
        host=conf("riak")["host"])

    MyBucket = MyClient.bucket(conf("riak")["bucket"])

    log_it("START", "bin-mining")
    for cube in mongo['cube'].find():
        try:
            slug = cube['slug']

            if cube_slug and cube_slug != slug:
                continue

            _sql = cube['sql']
            if _sql[-1] == ';':
                _sql = _sql[:-1]
            sql = u"""SELECT * FROM ({}) AS CUBE;""".format(_sql)

            connection = mongo['connection'].find_one({
                'slug': cube['connection']})['connection']

            log_it("CONNECT IN RELATION DATA BASE: {}".format(slug),
                   "bin-mining")
            e = create_engine(connection, pool_timeout=580, pool_size=100,
                              max_overflow=100, echo=True, echo_pool=True)
            connection = e.connect()

            resoverall = connection.execute(text(sql))

            log_it("LOAD DATA ON DATAWAREHOUSE: {}".format(slug),
                   "bin-mining")
            df = DataFrame(resoverall.fetchall())
            if df.empty:
                log_it('[warnning]Empty cube: {}!!'.format(cube),
                       "bin-mining")
                return
            df.columns = resoverall.keys()
            df.head()

            pdict = map(fix_render, df.to_dict(outtype='records'))

            MyBucket.new(slug, data='').store()
            MyBucket.new(u'{}-columns'.format(slug), data='').store()
            MyBucket.new(u'{}-connect'.format(slug), data='').store()
            MyBucket.new(u'{}-sql'.format(slug), data='').store()

            log_it("SAVE DATA (JSON) ON RIAK: {}".format(slug),
                   "bin-mining")
            MyBucket.new(slug, data=pdict).store()

            log_it("SAVE COLUMNS ON RIAK: {}".format(slug),
                   "bin-mining")
            MyBucket.new(u'{}-columns'.format(slug),
                         data=json.dumps([c for c in df.columns])).store()

            log_it("SAVE CONNECT ON RIAK: {}".format(slug),
                   "bin-mining")
            MyBucket.new(u'{}-connect'.format(slug), data=c).store()

            log_it("SAVE SQL ON RIAK: {}".format(slug),
                   "bin-mining")
            MyBucket.new(u'{}-sql'.format(slug), data=sql).store()

            cube['status'] = True
            cube['lastupdate'] = datetime.now()
            mongo['cube'].update({'slug': cube['slug']}, cube)

            log_it("CLEAN MEMORY: {}\n".format(slug), "bin-mining")
            del pdict, df
            gc.collect()
        except Exception, e:
            log_it(e, "bin-mining")

    log_it("END", "bin-mining")
    return True

if __name__ == "__main__":
    run()
<commit_msg>used sqlalchemy session on mining process sessionmaker<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from os import sys, path
import json
import riak
import gc
from datetime import datetime

from pandas import DataFrame
from sqlalchemy import create_engine
from sqlalchemy.sql import text
from sqlalchemy.orm import sessionmaker

sys.path.append(path.dirname(path.dirname(path.abspath(__file__))))
from utils import fix_render, conf, log_it

from bottle.ext.mongo import MongoPlugin

def run(cube_slug=None):
    mongo = MongoPlugin(
        uri=conf("mongodb")["uri"],
        db=conf("mongodb")["db"],
        json_mongo=True).get_mongo()

    MyClient = riak.RiakClient(
        protocol=conf("riak")["protocol"],
        http_port=conf("riak")["http_port"],
        host=conf("riak")["host"])

    MyBucket = MyClient.bucket(conf("riak")["bucket"])

    log_it("START", "bin-mining")
    for cube in mongo['cube'].find():
        try:
            slug = cube['slug']

            if cube_slug and cube_slug != slug:
                continue

            _sql = cube['sql']
            if _sql[-1] == ';':
                _sql = _sql[:-1]
            sql = u"""SELECT * FROM ({}) AS CUBE;""".format(_sql)

            connection = mongo['connection'].find_one({
                'slug': cube['connection']})['connection']

            log_it("CONNECT IN RELATION DATA BASE: {}".format(slug),
                   "bin-mining")
            e = create_engine(connection, pool_timeout=580, pool_size=100,
                              max_overflow=100)
            Session = sessionmaker(bind=e)
            session = Session()

            resoverall = session.execute(text(sql))

            log_it("LOAD DATA ON DATAWAREHOUSE: {}".format(slug),
                   "bin-mining")
            df = DataFrame(resoverall.fetchall())
            if df.empty:
                log_it('[warnning]Empty cube: {}!!'.format(cube),
                       "bin-mining")
                return
            df.columns = resoverall.keys()
            df.head()

            pdict = map(fix_render, df.to_dict(outtype='records'))

            MyBucket.new(slug, data='').store()
            MyBucket.new(u'{}-columns'.format(slug), data='').store()
            MyBucket.new(u'{}-connect'.format(slug), data='').store()
            MyBucket.new(u'{}-sql'.format(slug), data='').store()

            log_it("SAVE DATA (JSON) ON RIAK: {}".format(slug),
                   "bin-mining")
            MyBucket.new(slug, data=pdict).store()

            log_it("SAVE COLUMNS ON RIAK: {}".format(slug),
                   "bin-mining")
            MyBucket.new(u'{}-columns'.format(slug),
                         data=json.dumps([c for c in df.columns])).store()

            log_it("SAVE CONNECT ON RIAK: {}".format(slug),
                   "bin-mining")
            MyBucket.new(u'{}-connect'.format(slug), data=c).store()

            log_it("SAVE SQL ON RIAK: {}".format(slug),
                   "bin-mining")
            MyBucket.new(u'{}-sql'.format(slug), data=sql).store()

            cube['status'] = True
            cube['lastupdate'] = datetime.now()
            mongo['cube'].update({'slug': cube['slug']}, cube)

            log_it("CLEAN MEMORY: {}\n".format(slug), "bin-mining")
            del pdict, df
            gc.collect()
        except Exception, e:
            log_it(e, "bin-mining")

    log_it("END", "bin-mining")
    return True

if __name__ == "__main__":
    run()
<|endoftext|>


<commit_before>from django.db import models
from django.conf import settings
from django.core.validators import MinValueValidator

from django_countries.fields import CountryField
from uuslug import uuslug

from marketpulse.devices.models import Device
from marketpulse.geo.models import LocationBase
from marketpulse.main import get_currency_choices

class Activity(models.Model):
    """Model for activity types."""

    name = models.CharField(max_length=100)
    slug = models.SlugField(unique=True, max_length=255)

    class Meta:
        ordering = ['name']
        verbose_name = 'activity'
        verbose_name_plural = 'activities'

    def __unicode__(self):
        return self.name

    def save(self, *args, **kwargs):
        """Custom save method."""

        if not self.id:
            self.slug = uuslug(self.name, instance=self)
        super(Activity, self).save(*args, **kwargs)

class Location(LocationBase):
    """Model for contribution location."""

    address = models.CharField(max_length=120, blank=True, default='')
    shop_name = models.CharField(max_length=120)
    link = models.URLField(max_length=500, blank=True, default='')
    is_online = models.BooleanField(default=False)

    def __unicode__(self):
        return '{0}, {1}'.format(self.shop_name, self.country)

class Contribution(models.Model):
    """Model for contribution data."""

    user = models.ForeignKey(settings.AUTH_USER_MODEL, related_name='contributions')
    activity = models.ForeignKey(Activity, related_name='contributions')
    location = models.ForeignKey(Location, related_name='contributions', null=True)
    device = models.ForeignKey(Device, related_name='contributions', null=True)
    created_on = models.DateTimeField(auto_now_add=True)
    updated_on = models.DateTimeField(auto_now=True)
    comment = models.TextField(blank=True, default='')
    availability = models.BooleanField(default=True)

    def __unicode__(self):
        return '{0}, {1}'.format(self.user, self.activity)

class Carrier(models.Model):
    name = models.CharField(max_length=128)
    parent_operator = models.CharField(max_length=128, blank=True, default='')
    country = CountryField(null=True, blank=True)

    def __unicode__(self):
        return self.name

class Plan(models.Model):
    """Mobile phone plan information."""

    contribution = models.ForeignKey(Contribution, related_name='plans', null=True)
    has_plan = models.BooleanField(default=False)
    duration = models.PositiveIntegerField(blank=True, null=True)
    description = models.TextField(default='', blank=True)
    amount = models.FloatField(blank=True, null=True, validators=[MinValueValidator(0.0)])
    currency = models.CharField(max_length=128, choices=get_currency_choices(), default='')
    carrier = models.ForeignKey(Carrier, related_name='carriers', null=True, blank=True)
    monthly_fee = models.FloatField(blank=True, null=True, validators=[MinValueValidator(0.0)])
<commit_msg>Fix unicode method in carriers model.<commit_after>from django.db import models
from django.conf import settings
from django.core.validators import MinValueValidator

from django_countries import countries
from django_countries.fields import CountryField
from uuslug import uuslug

from marketpulse.devices.models import Device
from marketpulse.geo.models import LocationBase
from marketpulse.main import get_currency_choices

class Activity(models.Model):
    """Model for activity types."""

    name = models.CharField(max_length=100)
    slug = models.SlugField(unique=True, max_length=255)

    class Meta:
        ordering = ['name']
        verbose_name = 'activity'
        verbose_name_plural = 'activities'

    def __unicode__(self):
        return self.name

    def save(self, *args, **kwargs):
        """Custom save method."""

        if not self.id:
            self.slug = uuslug(self.name, instance=self)
        super(Activity, self).save(*args, **kwargs)

class Location(LocationBase):
    """Model for contribution location."""

    address = models.CharField(max_length=120, blank=True, default='')
    shop_name = models.CharField(max_length=120)
    link = models.URLField(max_length=500, blank=True, default='')
    is_online = models.BooleanField(default=False)

    def __unicode__(self):
        return '{0}, {1}'.format(self.shop_name, self.country)

class Contribution(models.Model):
    """Model for contribution data."""

    user = models.ForeignKey(settings.AUTH_USER_MODEL, related_name='contributions')
    activity = models.ForeignKey(Activity, related_name='contributions')
    location = models.ForeignKey(Location, related_name='contributions', null=True)
    device = models.ForeignKey(Device, related_name='contributions', null=True)
    created_on = models.DateTimeField(auto_now_add=True)
    updated_on = models.DateTimeField(auto_now=True)
    comment = models.TextField(blank=True, default='')
    availability = models.BooleanField(default=True)

    def __unicode__(self):
        return '{0}, {1}'.format(self.user, self.activity)

class Carrier(models.Model):
    name = models.CharField(max_length=128)
    parent_operator = models.CharField(max_length=128, blank=True, default='')
    country = CountryField(null=True, blank=True)

    def __unicode__(self):
        all_countries = dict(countries)
        country = self.country
        if self.country.code in all_countries:
            country = all_countries[self.country.code]

        return '{0}, {1}, {2}'.format(self.name, country, self.parent_operator)

class Plan(models.Model):
    """Mobile phone plan information."""

    contribution = models.ForeignKey(Contribution, related_name='plans', null=True)
    has_plan = models.BooleanField(default=False)
    duration = models.PositiveIntegerField(blank=True, null=True)
    description = models.TextField(default='', blank=True)
    amount = models.FloatField(blank=True, null=True, validators=[MinValueValidator(0.0)])
    currency = models.CharField(max_length=128, choices=get_currency_choices(), default='')
    carrier = models.ForeignKey(Carrier, related_name='carriers', null=True, blank=True)
    monthly_fee = models.FloatField(blank=True, null=True, validators=[MinValueValidator(0.0)])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Utility classes and values used for marshalling and unmarshalling objects to
and from primitive types.

.. warning::

    This module is treated as private API.
    Users should not need to use this module directly.
"""

from __future__ import unicode_literals

from marshmallow.utils import missing
from marshmallow.compat import text_type, iteritems
from marshmallow.exceptions import (
    ValidationError,
)

__all__ = [
    'Marshaller',
    'Unmarshaller',
]

class ErrorStore(object):

    def __init__(self):
        #: Dictionary of errors stored during serialization
        self.errors = {}
        #: List of `Field` objects which have validation errors
        self.error_fields = []
        #: List of field_names which have validation errors
        self.error_field_names = []
        #: True while (de)serializing a collection
        self._pending = False

    def reset_errors(self):
        self.errors = {}
        self.error_field_names = []
        self.error_fields = []

    def get_errors(self, index=None):
        if index is not None:
            errors = self.errors.get(index, {})
            self.errors[index] = errors
        else:
            errors = self.errors
        return errors

    def call_and_store(self, getter_func, data, field_name, field_obj, index=None):
        """Call ``getter_func`` with ``data`` as its argument, and store any `ValidationErrors`.

        :param callable getter_func: Function for getting the serialized/deserialized
            value from ``data``.
        :param data: The data passed to ``getter_func``.
        :param str field_name: Field name.
        :param FieldABC field_obj: Field object that performs the
            serialization/deserialization behavior.
        :param int index: Index of the item being validated, if validating a collection,
            otherwise `None`.
        """
        try:
            value = getter_func(data)
        except ValidationError as err:  # Store validation errors
            self.error_fields.append(field_obj)
            self.error_field_names.append(field_name)
            errors = self.get_errors(index=index)
            # Warning: Mutation!
            if isinstance(err.messages, dict):
                errors[field_name] = err.messages
            else:
                errors.setdefault(field_name, []).extend(err.messages)
            # When a Nested field fails validation, the marshalled data is stored
            # on the ValidationError's data attribute
            value = err.data or missing
        return value

class Marshaller(ErrorStore):
    """Callable class responsible for serializing data and storing errors.

    :param str prefix: Optional prefix that will be prepended to all the
        serialized field names.
    """
    def __init__(self, prefix=''):
        self.prefix = prefix
        ErrorStore.__init__(self)

    def serialize(self, obj, fields_dict, many=False, strict=False,
                  accessor=None, dict_class=dict, index_errors=True, index=None):
        """Takes raw data (a dict, list, or other object) and a dict of
        fields to output and serializes the data based on those fields.

        :param obj: The actual object(s) from which the fields are taken from
        :param dict fields_dict: Mapping of field names to :class:`Field` objects.
        :param bool many: Set to `True` if ``data`` should be serialized as
            a collection.
        :param bool strict: If `True`, raise errors if invalid data are passed in
            instead of failing silently and storing the errors.
        :param callable accessor: Function to use for getting values from ``obj``.
        :param type dict_class: Dictionary class used to construct the output.
        :param bool index_errors: Whether to store the index of invalid items in
            ``self.errors`` when ``many=True``.
        :param int index: Index of the item being serialized (for storing errors) if
            serializing a collection, otherwise `None`.
        :return: A dictionary of the marshalled data

        .. versionchanged:: 1.0.0
            Renamed from ``marshal``.
        """
        # Reset errors dict if not serializing a collection
        if not self._pending:
            self.reset_errors()
        if many and obj is not None:
            self._pending = True
            ret = [self.serialize(d, fields_dict, many=False, strict=strict,
                                    dict_class=dict_class, accessor=accessor,
                                    index=idx, index_errors=index_errors)
                    for idx, d in enumerate(obj)]
            self._pending = False
            return ret
        items = []
        for attr_name, field_obj in iteritems(fields_dict):
            if getattr(field_obj, 'load_only', False):
                continue
            if not self.prefix:
                key = attr_name
            else:
                key = ''.join([self.prefix, attr_name])
            getter = lambda d: field_obj.serialize(attr_name, d, accessor=accessor)
            value = self.call_and_store(
                getter_func=getter,
                data=obj,
                field_name=key,
                field_obj=field_obj,
                index=(index if index_errors else None)
            )
            if value is missing:
                continue
            items.append((key, value))
        if self.errors and strict:
            raise ValidationError(
                self.errors,
                field_names=self.error_field_names,
                fields=self.error_fields
            )
        return dict_class(items)

    # Make an instance callable
    __call__ = serialize

# Key used for schema-level validation errors
SCHEMA = '_schema'

class Unmarshaller(ErrorStore):
    """Callable class responsible for deserializing data and storing errors.

    .. versionadded:: 1.0.0
    """

    default_schema_validation_error = 'Invalid data.'

    def _run_validator(self, validator_func, output,
            original_data, fields_dict, index=None,
            strict=False, many=False, pass_original=False):
        try:
            if pass_original:  # Pass original, raw data (before unmarshalling)
                res = validator_func(output, original_data)
            else:
                res = validator_func(output)
            if res is False:
                raise ValidationError(self.default_schema_validation_error)
        except ValidationError as err:
            errors = self.get_errors(index=index)
            # Store or reraise errors
            if err.field_names:
                field_names = err.field_names
                field_objs = [fields_dict[each] if each in fields_dict else None for each in field_names]
            else:
                field_names = [SCHEMA]
                field_objs = []
            for field_name in field_names:
                if isinstance(err.messages, (list, tuple)):
                    # self.errors[field_name] may be a dict if schemas are nested
                    if isinstance(errors.get(field_name), dict):
                        errors[field_name].setdefault(
                            SCHEMA, []
                        ).extend(err.messages)
                    else:
                        errors.setdefault(field_name, []).extend(err.messages)
                elif isinstance(err.messages, dict):
                    errors.setdefault(field_name, []).append(err.messages)
                else:
                    errors.setdefault(field_name, []).append(text_type(err))
            if strict:
                raise ValidationError(
                    self.errors,
                    fields=field_objs,
                    field_names=field_names
                )

    def _validate(self, validators, output, original_data, fields_dict, index=None, strict=False):
        """Perform schema-level validation. Stores errors if ``strict`` is `False`.
        """
        for validator_func in validators:
            self._run_validator(validator_func, output, original_data, fields_dict,
                    index=index, strict=strict)
        return output

    def deserialize(self, data, fields_dict, many=False, validators=None,
            preprocess=None, postprocess=None, strict=False, dict_class=dict,
            index_errors=True, index=None):
        """Deserialize ``data`` based on the schema defined by ``fields_dict``.

        :param dict data: The data to deserialize.
        :param dict fields_dict: Mapping of field names to :class:`Field` objects.
        :param bool many: Set to `True` if ``data`` should be deserialized as
            a collection.
        :param list validators: List of validation functions to apply to the
            deserialized dictionary.
        :param list preprocess: List of pre-processing functions.
        :param bool strict: If `True`, raise errors if invalid data are passed in
            instead of failing silently and storing the errors.
        :param type dict_class: Dictionary class used to construct the output.
        :param bool index_errors: Whether to store the index of invalid items in
            ``self.errors`` when ``many=True``.
        :param int index: Index of the item being serialized (for storing errors) if
            serializing a collection, otherwise `None`.
        :return: A dictionary of the deserialized data.
        """
        # Reset errors if not deserializing a collection
        if not self._pending:
            self.reset_errors()
        if many and data is not None:
            self._pending = True
            ret = [self.deserialize(d, fields_dict, many=False,
                        validators=validators, preprocess=preprocess,
                        strict=strict, dict_class=dict_class,
                        index=idx, index_errors=index_errors)
                    for idx, d in enumerate(data)]
            self._pending = False
            return ret
        original_data = data
        if data is not None:
            items = []
            for attr_name, field_obj in iteritems(fields_dict):
                if field_obj.dump_only:
                    continue
                try:
                    raw_value = data.get(attr_name, missing)
                except AttributeError:  # Input data is not a dict
                    msg = 'Data must be a dict, got a {0}'.format(data.__class__.__name__)
                    errors = self.get_errors(index=index)
                    if strict:
                        raise ValidationError(
                            msg,
                            field_names=[SCHEMA],
                            fields=[]
                        )
                    else:
                        errors = self.get_errors()
                        errors.setdefault(SCHEMA, []).append(msg)
                        # Input data type is incorrect, so we can bail out early
                        break
                field_name = attr_name
                if raw_value is missing and field_obj.load_from:
                    field_name = field_obj.load_from
                    raw_value = data.get(field_obj.load_from, missing)
                if raw_value is missing:
                    _miss = field_obj.missing
                    raw_value = _miss() if callable(_miss) else _miss
                if raw_value is missing and not field_obj.required:
                    continue

                getter = lambda val: field_obj.deserialize(
                    val,
                    field_obj.load_from or attr_name,
                    data
                )
                value = self.call_and_store(
                    getter_func=getter,
                    data=raw_value,
                    field_name=field_name,
                    field_obj=field_obj,
                    index=(index if index_errors else None)
                )
                if value is not missing:
                    key = fields_dict[attr_name].attribute or attr_name
                    items.append((key, value))
            ret = dict_class(items)
        else:
            ret = None

        if preprocess:
            preprocess = preprocess or []
            for func in preprocess:
                ret = func(ret)
        if validators:
            validators = validators or []
            ret = self._validate(validators, ret, original_data, fields_dict=fields_dict,
                                 strict=strict, index=(index if index_errors else None))
        if self.errors and strict:
            raise ValidationError(
                self.errors,
                field_names=self.error_field_names,
                fields=self.error_fields
            )
        return ret

    # Make an instance callable
    __call__ = deserialize
<commit_msg>Fix flake error<commit_after># -*- coding: utf-8 -*-
"""Utility classes and values used for marshalling and unmarshalling objects to
and from primitive types.

.. warning::

    This module is treated as private API.
    Users should not need to use this module directly.
"""

from __future__ import unicode_literals

from marshmallow.utils import missing
from marshmallow.compat import text_type, iteritems
from marshmallow.exceptions import (
    ValidationError,
)

__all__ = [
    'Marshaller',
    'Unmarshaller',
]

class ErrorStore(object):

    def __init__(self):
        #: Dictionary of errors stored during serialization
        self.errors = {}
        #: List of `Field` objects which have validation errors
        self.error_fields = []
        #: List of field_names which have validation errors
        self.error_field_names = []
        #: True while (de)serializing a collection
        self._pending = False

    def reset_errors(self):
        self.errors = {}
        self.error_field_names = []
        self.error_fields = []

    def get_errors(self, index=None):
        if index is not None:
            errors = self.errors.get(index, {})
            self.errors[index] = errors
        else:
            errors = self.errors
        return errors

    def call_and_store(self, getter_func, data, field_name, field_obj, index=None):
        """Call ``getter_func`` with ``data`` as its argument, and store any `ValidationErrors`.

        :param callable getter_func: Function for getting the serialized/deserialized
            value from ``data``.
        :param data: The data passed to ``getter_func``.
        :param str field_name: Field name.
        :param FieldABC field_obj: Field object that performs the
            serialization/deserialization behavior.
        :param int index: Index of the item being validated, if validating a collection,
            otherwise `None`.
        """
        try:
            value = getter_func(data)
        except ValidationError as err:  # Store validation errors
            self.error_fields.append(field_obj)
            self.error_field_names.append(field_name)
            errors = self.get_errors(index=index)
            # Warning: Mutation!
            if isinstance(err.messages, dict):
                errors[field_name] = err.messages
            else:
                errors.setdefault(field_name, []).extend(err.messages)
            # When a Nested field fails validation, the marshalled data is stored
            # on the ValidationError's data attribute
            value = err.data or missing
        return value

class Marshaller(ErrorStore):
    """Callable class responsible for serializing data and storing errors.

    :param str prefix: Optional prefix that will be prepended to all the
        serialized field names.
    """
    def __init__(self, prefix=''):
        self.prefix = prefix
        ErrorStore.__init__(self)

    def serialize(self, obj, fields_dict, many=False, strict=False,
                  accessor=None, dict_class=dict, index_errors=True, index=None):
        """Takes raw data (a dict, list, or other object) and a dict of
        fields to output and serializes the data based on those fields.

        :param obj: The actual object(s) from which the fields are taken from
        :param dict fields_dict: Mapping of field names to :class:`Field` objects.
        :param bool many: Set to `True` if ``data`` should be serialized as
            a collection.
        :param bool strict: If `True`, raise errors if invalid data are passed in
            instead of failing silently and storing the errors.
        :param callable accessor: Function to use for getting values from ``obj``.
        :param type dict_class: Dictionary class used to construct the output.
        :param bool index_errors: Whether to store the index of invalid items in
            ``self.errors`` when ``many=True``.
        :param int index: Index of the item being serialized (for storing errors) if
            serializing a collection, otherwise `None`.
        :return: A dictionary of the marshalled data

        .. versionchanged:: 1.0.0
            Renamed from ``marshal``.
        """
        # Reset errors dict if not serializing a collection
        if not self._pending:
            self.reset_errors()
        if many and obj is not None:
            self._pending = True
            ret = [self.serialize(d, fields_dict, many=False, strict=strict,
                                    dict_class=dict_class, accessor=accessor,
                                    index=idx, index_errors=index_errors)
                    for idx, d in enumerate(obj)]
            self._pending = False
            return ret
        items = []
        for attr_name, field_obj in iteritems(fields_dict):
            if getattr(field_obj, 'load_only', False):
                continue
            if not self.prefix:
                key = attr_name
            else:
                key = ''.join([self.prefix, attr_name])
            getter = lambda d: field_obj.serialize(attr_name, d, accessor=accessor)
            value = self.call_and_store(
                getter_func=getter,
                data=obj,
                field_name=key,
                field_obj=field_obj,
                index=(index if index_errors else None)
            )
            if value is missing:
                continue
            items.append((key, value))
        if self.errors and strict:
            raise ValidationError(
                self.errors,
                field_names=self.error_field_names,
                fields=self.error_fields
            )
        return dict_class(items)

    # Make an instance callable
    __call__ = serialize

# Key used for schema-level validation errors
SCHEMA = '_schema'

class Unmarshaller(ErrorStore):
    """Callable class responsible for deserializing data and storing errors.

    .. versionadded:: 1.0.0
    """

    default_schema_validation_error = 'Invalid data.'

    def _run_validator(self, validator_func, output,
            original_data, fields_dict, index=None,
            strict=False, many=False, pass_original=False):
        try:
            if pass_original:  # Pass original, raw data (before unmarshalling)
                res = validator_func(output, original_data)
            else:
                res = validator_func(output)
            if res is False:
                raise ValidationError(self.default_schema_validation_error)
        except ValidationError as err:
            errors = self.get_errors(index=index)
            # Store or reraise errors
            if err.field_names:
                field_names = err.field_names
                field_objs = [fields_dict[each] if each in fields_dict else None
                              for each in field_names]
            else:
                field_names = [SCHEMA]
                field_objs = []
            for field_name in field_names:
                if isinstance(err.messages, (list, tuple)):
                    # self.errors[field_name] may be a dict if schemas are nested
                    if isinstance(errors.get(field_name), dict):
                        errors[field_name].setdefault(
                            SCHEMA, []
                        ).extend(err.messages)
                    else:
                        errors.setdefault(field_name, []).extend(err.messages)
                elif isinstance(err.messages, dict):
                    errors.setdefault(field_name, []).append(err.messages)
                else:
                    errors.setdefault(field_name, []).append(text_type(err))
            if strict:
                raise ValidationError(
                    self.errors,
                    fields=field_objs,
                    field_names=field_names
                )

    def _validate(self, validators, output, original_data, fields_dict, index=None, strict=False):
        """Perform schema-level validation. Stores errors if ``strict`` is `False`.
        """
        for validator_func in validators:
            self._run_validator(validator_func, output, original_data, fields_dict,
                    index=index, strict=strict)
        return output

    def deserialize(self, data, fields_dict, many=False, validators=None,
            preprocess=None, postprocess=None, strict=False, dict_class=dict,
            index_errors=True, index=None):
        """Deserialize ``data`` based on the schema defined by ``fields_dict``.

        :param dict data: The data to deserialize.
        :param dict fields_dict: Mapping of field names to :class:`Field` objects.
        :param bool many: Set to `True` if ``data`` should be deserialized as
            a collection.
        :param list validators: List of validation functions to apply to the
            deserialized dictionary.
        :param list preprocess: List of pre-processing functions.
        :param bool strict: If `True`, raise errors if invalid data are passed in
            instead of failing silently and storing the errors.
        :param type dict_class: Dictionary class used to construct the output.
        :param bool index_errors: Whether to store the index of invalid items in
            ``self.errors`` when ``many=True``.
        :param int index: Index of the item being serialized (for storing errors) if
            serializing a collection, otherwise `None`.
        :return: A dictionary of the deserialized data.
        """
        # Reset errors if not deserializing a collection
        if not self._pending:
            self.reset_errors()
        if many and data is not None:
            self._pending = True
            ret = [self.deserialize(d, fields_dict, many=False,
                        validators=validators, preprocess=preprocess,
                        strict=strict, dict_class=dict_class,
                        index=idx, index_errors=index_errors)
                    for idx, d in enumerate(data)]
            self._pending = False
            return ret
        original_data = data
        if data is not None:
            items = []
            for attr_name, field_obj in iteritems(fields_dict):
                if field_obj.dump_only:
                    continue
                try:
                    raw_value = data.get(attr_name, missing)
                except AttributeError:  # Input data is not a dict
                    msg = 'Data must be a dict, got a {0}'.format(data.__class__.__name__)
                    errors = self.get_errors(index=index)
                    if strict:
                        raise ValidationError(
                            msg,
                            field_names=[SCHEMA],
                            fields=[]
                        )
                    else:
                        errors = self.get_errors()
                        errors.setdefault(SCHEMA, []).append(msg)
                        # Input data type is incorrect, so we can bail out early
                        break
                field_name = attr_name
                if raw_value is missing and field_obj.load_from:
                    field_name = field_obj.load_from
                    raw_value = data.get(field_obj.load_from, missing)
                if raw_value is missing:
                    _miss = field_obj.missing
                    raw_value = _miss() if callable(_miss) else _miss
                if raw_value is missing and not field_obj.required:
                    continue

                getter = lambda val: field_obj.deserialize(
                    val,
                    field_obj.load_from or attr_name,
                    data
                )
                value = self.call_and_store(
                    getter_func=getter,
                    data=raw_value,
                    field_name=field_name,
                    field_obj=field_obj,
                    index=(index if index_errors else None)
                )
                if value is not missing:
                    key = fields_dict[attr_name].attribute or attr_name
                    items.append((key, value))
            ret = dict_class(items)
        else:
            ret = None

        if preprocess:
            preprocess = preprocess or []
            for func in preprocess:
                ret = func(ret)
        if validators:
            validators = validators or []
            ret = self._validate(validators, ret, original_data, fields_dict=fields_dict,
                                 strict=strict, index=(index if index_errors else None))
        if self.errors and strict:
            raise ValidationError(
                self.errors,
                field_names=self.error_field_names,
                fields=self.error_fields
            )
        return ret

    # Make an instance callable
    __call__ = deserialize
<|endoftext|>