<commit_before>from backend.Node import Node
import math
class AI:
    root = None
    priorityMap = {} # Assigns weight to each node based on depth and children per node
    moveMap = {} # possible moves for each node
    def __init__(self,r):
        self.root = r
        print(self.moveMap)
    # @RN March 15 2016
    # Sets priority for each node based on depth / children of node
    # Nodes without children that are lower in the tree are prioritized higher than higher nodes with children
    def generatePriorityMap(self, root,depth=0):
        if root:
            self.priorityMap[root.ID] = (depth / (len(root.children) + 1))
            for ids, child in root.children.items():
                self.generatePriorityMap(child,depth+1)

    # @RN March 15 2016
    # Generates possible moves for each node based segmented by an angle
    def generateMoveMap(self, root, radius, degree=10, segments=0,radian=False):
        if not radian:
            degree = math.radians(degree)
        if segments == 0:
            segments = int(round(2*math.pi / degree,0))
        if root:
            self.moveMap[root.ID] = []
            for i in range(segments):
                self.moveMap[root.ID].append((round(root.x + radius*math.cos(i*degree),0),round(root.y +
                    radius*math.sin(i*degree),0)))
            for ids, child in root.children.items():
                self.generateMoveMap(child,radius,degree,segments,radian)

<commit_msg>Added Minimax and a getMove function. They may not be correct yet; need to talk to shrub about exactly what moveMap is.<commit_after>from Node import Node
import math
import sys
class AI:
    root = None
    priorityMap = {} # Assigns weight to each node based on depth and children per node
    moveMap = {} # possible moves for each node
    def __init__(self,r):
        self.root = r
        print(self.moveMap)
    # @RN March 15 2016
    # Sets priority for each node based on depth / children of node
    # Nodes without children that are lower in the tree are prioritized higher than higher nodes with children
    def generatePriorityMap(self, root,depth=0):
        if root:
            self.priorityMap[root.ID] = (depth / (len(root.children) + 1))
            for ids, child in root.children.items():
                self.generatePriorityMap(child,depth+1)

    # @RN March 15 2016
    # Generates possible moves for each node based segmented by an angle
    def generateMoveMap(self, root, radius, degree=10, segments=0,radian=False):
        if not radian:
            degree = math.radians(degree)
        if segments == 0:
            segments = int(round(2*math.pi / degree,0))
        if root:
            self.moveMap[root.ID] = []
            for i in range(segments):
                self.moveMap[root.ID].append((round(root.x + radius*math.cos(i*degree),0),round(root.y +
                    radius*math.sin(i*degree),0)))
            for ids, child in root.children.items():
                self.generateMoveMap(child,radius,degree,segments,radian)

    #returns the value of a move
    #these nodes are NOT game nodes, they are gamestates.
    def minimax(board, depth, pnum):
        if depth == 0 or len(node.children) == 0:
            return heuristic(board)

        if pnum == 0: #max
            bestvalue = sys.float_info.max
            for move in moveMap: #probably wrong
                value = minimax(move, depth - 1, 1)
                bestvalue = max(value, bestvalue)
        else: #min
            bestvalue = sys.float_info.min
            for move in moveMap: #ditto
                value = minimax(move, depth - 1, 0)
                bestvalue = min(bestvalue, value)
        return value

    #bs is a board state (a list of roots)
    def heuristic(bs):
        return 0

    #thebs is the current boar state
    def getMove(thebs, depth, pnum):
        bestMove = None
        bestMoveValue = sys.float_info.min
        for move in moveMap: #might be wrong
            value = minimax(thebs, depth, pnum)
            if value >= bestMoveValue:
                bestMoveValue = value
                bestMove = move
        return bestMove<|endoftext|>


<commit_before>'''
Created by:
Dori Dick
College of Earth, Ocean and Atmospheric Sciences
Oregon State Univeristy

Created on: 29 July 2012
Last modified: 15 May 2013

Description:
This script exports a feature class containing spatially referenced genetic into the text file
format required by Genepop. See http://genepop.curtin.edu.au/ for more information on Genepop.

Required Inputs:
An existing Feature Class containing spatially referenced genetic data (i.e. co-dominant microsatellite data).
An Attribute Field within the Feature Class that can be used to designate different populations.
An Output File Location  and file name for the text file

Optional Inputs:
A Where Clause to identify only those rows with genetic data in the Feature Class.  Uses a SQL Expression.
NOTE: This parameter is optional and was included because some data sets may have individual IDs based
on more than just genetics (i.e. photo-id).

Script Outputs:
A text file formatted to meet the required input file format for Genepop.

NOTE:  Keep a record of the Loci names and their order as this script uses generic locus names
(e.g., Locus1_A1, Locus1_A2, Locus2_A1, Locus2_A2 etc.)

This script was developed to work with ArcGIS 10.1 and Python 2.7 installed during ArcGIS installation.
'''

import arcpy
import os
import sys
import binascii
import collections
from datetime import datetime

# local imports
import utils
import config

def main(input_features=None, where_clause=None, order_by=None,  output_name=None,
         mode=config.settings.mode):

        utils.msg("Executing ExportToGenepop.")

        # Create and open the Output text file to which the data will be written
        with open(output_name + ".txt", "w") as output_file:
            utils.msg("file open for writing")

            # Create the Genepop header information required for the text file.
            # NOTE: Line 1 includes user inputed information about the data in the file.  Lines 2-15 list the generic Loci names
            #FIXME: What we want is to have the script write out the acutal Loci names of the input file.  This way it will be specific to the user and will not be
            #fixed in naming convention or the number of loci
            output_file.write("Export to Genepop from the feature class: " + input_features + ". Export occurred on: " + str(datetime.now()) + "\n")
            output_file.write("Locus1\n")
            output_file.write("Locus2\n")
            output_file.write("Locus3\n")
            output_file.write("Locus4\n")
            output_file.write("Locus5\n")
            output_file.write("Locus6\n")
            output_file.write("Locus7\n")
            output_file.write("Locus8\n")
            output_file.write("Locus9\n")
            output_file.write("Locus10\n")
            output_file.write("Locus11\n")
            output_file.write("Locus12\n")
            output_file.write("Locus13\n")
            output_file.write("Locus14\n")

            # Creating a search cursor that will move through the Input Feature Class row by row (Parameter1),
            # specifying only genetic records (Parameter2, if specified by the used) and ordering the data by a population
            # field (Parameter3) in ascending order
            rows = arcpy.SearchCursor(input_features, where_clause, "", "", order_by + " A")

            # Defining an empty variable as AreaName
            AreaName = ""

            # Creating a list of the generic Loci names
            LocusNames = ["Locus1_A1", "Locus1_A2", "Locus2_A1", "Locus2_A2", "Locus3_A1", "Locus3_A2", "Locus4_A1",\
                    "Locus4_A2", "Locus5_A1", "Locus5_A2", "Locus6_A1", "Locus6_A2", "Locus7_A1", "Locus7_A2", "Locus8_A1", \
                    "Locus8_A2", "Locus9_A1", "Locus9_A2", "Locus10_A1", "Locus10_A2", "Locus11_A1", "Locus11_A2", \
                    "Locus12_A1", "Locus12_A2", "Locus13_A1", "Locus13_A2", "Locus14_A1", "Locus14_A2"]

            # For loop used to move through each row of data and pull the user defined Parameter4 value for each of the necessary fields
            for row in rows:
                    # Defining a variable for the Parameter3 value
                    temp = str(row.getValue(order_by))
                    # Set a boolean statement
                    HaveZero = False

            # If temp is not equal to the AreaName
            if temp <> AreaName:
                    # Write "Pop" to the output text file and go to new line
                    output_file.write("Pop" + "\n")
                    # Re-define AreaName = temp
                    AreaName = temp
            # Set a boolean statement
            OddName = False
            # Write to the text file the IndividualID-Parameter3 Value
            output_name.write(str(row.Individual_ID) + "-" + str(row.getValue(order_by)) +"," + " ")

           # For loop used to move through the LocusNames list
            for Name in LocusNames:
                    # Defining a variable for the LocusValue
                    LocusValue = str(row.getValue(Name))
                    # If the LocusValue is equivalent to a 0 or a 00
                    if LocusValue == "0" or LocusValue == "00":
                            # then HaveZero is True
                            HaveZero = True
                            # Make the LocusValue 000
                            LocusValue = str("000")

            # Write to the output text file the LocusValue
            output_name.write(LocusValue)

            # If OddName is equivalent to True
            if OddName == True:
                    # Write to the output text file a space
                    output_name.write(" ")

                    # Update flag - if OddName is equivalent to False
                    if OddName == False:
                            # Set OddName to true
                            OddName = True

            else:
                    # Re-setting OddName to False
                    OddName = False

            # Write to output text file a new line
            output_name.write("\n")

            # Go to the next row
            row += 1
<commit_msg>Fix whitespace issues.<commit_after>'''
Created by:
Dori Dick
College of Earth, Ocean and Atmospheric Sciences
Oregon State Univeristy

Created on: 29 July 2012
Last modified: 15 May 2013

Description:
This script exports a feature class containing spatially referenced genetic into the text file
format required by Genepop. See http://genepop.curtin.edu.au/ for more information on Genepop.

Required Inputs:
An existing Feature Class containing spatially referenced genetic data (i.e. co-dominant microsatellite data).
An Attribute Field within the Feature Class that can be used to designate different populations.
An Output File Location  and file name for the text file

Optional Inputs:
A Where Clause to identify only those rows with genetic data in the Feature Class.  Uses a SQL Expression.
NOTE: This parameter is optional and was included because some data sets may have individual IDs based
on more than just genetics (i.e. photo-id).

Script Outputs:
A text file formatted to meet the required input file format for Genepop.

NOTE:  Keep a record of the Loci names and their order as this script uses generic locus names
(e.g., Locus1_A1, Locus1_A2, Locus2_A1, Locus2_A2 etc.)

This script was developed to work with ArcGIS 10.1 and Python 2.7 installed during ArcGIS installation.
'''

import arcpy
import os
import sys
import binascii
import collections
from datetime import datetime

# local imports
import utils
import config

def main(input_features=None, where_clause=None, order_by=None,  output_name=None,
         mode=config.settings.mode):

    utils.msg("Executing ExportToGenepop.")

    # Create and open the Output text file to which the data will be written
    with open(output_name + ".txt", "w") as output_file:
        utils.msg("file open for writing")

        # Create the Genepop header information required for the text file.
        # NOTE: Line 1 includes user inputed information about the data in the file.  Lines 2-15 list the generic Loci names
        #FIXME: What we want is to have the script write out the acutal Loci names of the input file.  This way it will be specific to the user and will not be
        #fixed in naming convention or the number of loci
        output_file.write("Export to Genepop from the feature class: " + input_features + ". Export occurred on: " + str(datetime.now()) + "\n")
        output_file.write("Locus1\n")
        output_file.write("Locus2\n")
        output_file.write("Locus3\n")
        output_file.write("Locus4\n")
        output_file.write("Locus5\n")
        output_file.write("Locus6\n")
        output_file.write("Locus7\n")
        output_file.write("Locus8\n")
        output_file.write("Locus9\n")
        output_file.write("Locus10\n")
        output_file.write("Locus11\n")
        output_file.write("Locus12\n")
        output_file.write("Locus13\n")
        output_file.write("Locus14\n")

        # Creating a search cursor that will move through the Input Feature Class row by row (Parameter1),
        # specifying only genetic records (Parameter2, if specified by the used) and ordering the data by a population
        # field (Parameter3) in ascending order
        rows = arcpy.SearchCursor(input_features, where_clause, "", "", order_by + " A")

        # Defining an empty variable as AreaName
        AreaName = ""

        # Creating a list of the generic Loci names
        LocusNames = ["Locus1_A1", "Locus1_A2", "Locus2_A1", "Locus2_A2", "Locus3_A1", "Locus3_A2", "Locus4_A1",\
                "Locus4_A2", "Locus5_A1", "Locus5_A2", "Locus6_A1", "Locus6_A2", "Locus7_A1", "Locus7_A2", "Locus8_A1", \
                "Locus8_A2", "Locus9_A1", "Locus9_A2", "Locus10_A1", "Locus10_A2", "Locus11_A1", "Locus11_A2", \
                "Locus12_A1", "Locus12_A2", "Locus13_A1", "Locus13_A2", "Locus14_A1", "Locus14_A2"]

        # For loop used to move through each row of data and pull the user defined Parameter4 value for each of the necessary fields
        for row in rows:
                # Defining a variable for the Parameter3 value
                temp = str(row.getValue(order_by))
                # Set a boolean statement
                HaveZero = False

        # If temp is not equal to the AreaName
        if temp <> AreaName:
                # Write "Pop" to the output text file and go to new line
                output_file.write("Pop" + "\n")
                # Re-define AreaName = temp
                AreaName = temp
        # Set a boolean statement
        OddName = False
        # Write to the text file the IndividualID-Parameter3 Value
        output_name.write(str(row.Individual_ID) + "-" + str(row.getValue(order_by)) +"," + " ")

       # For loop used to move through the LocusNames list
        for Name in LocusNames:
                # Defining a variable for the LocusValue
                LocusValue = str(row.getValue(Name))
                # If the LocusValue is equivalent to a 0 or a 00
                if LocusValue == "0" or LocusValue == "00":
                        # then HaveZero is True
                        HaveZero = True
                        # Make the LocusValue 000
                        LocusValue = str("000")

        # Write to the output text file the LocusValue
        output_name.write(LocusValue)

        # If OddName is equivalent to True
        if OddName == True:
                # Write to the output text file a space
                output_name.write(" ")

                # Update flag - if OddName is equivalent to False
                if OddName == False:
                        # Set OddName to true
                        OddName = True

        else:
                # Re-setting OddName to False
                OddName = False

        # Write to output text file a new line
        output_name.write("\n")

        # Go to the next row
        row += 1
<|endoftext|>


<commit_before>import zpt
import re

METHODS = ['get', 'post', 'put', 'patch', 'delete', 'head']
PARAM_PATTERN = re.compile(r'(\{[^\}]+\})')

class RestHandler(object):

    def __init__(self):
        self.web_endpoint = None
        self.data_layer_endpoint = None
        self.data_layer_params = None
        self.methods_allowed = METHODS
        self.unauthorized_status = 401
        self.unauthorized_code = 1019
        self.unauthorized_text = 'unauthorized access to this resource'

    def __getattribute__(self, name):

        try:
            return object.__getattribute__(self, name)
        except Exception as e:

            if name in self.methods_allowed:
                return self.dispatch
            elif name in ['{}_callback'.format(method) for method in self.methods_allowed]:
                return self.default_callback

        super(RestHandler, self).__getattribute__(name)

    def __setattribute__(self, name, value):

        if name == 'methods_allowed':
            return [method for method in self.__methods_allowed if method in METHODS]

        super(RestHandler, self).__setattribute__(name, value)

    def authorize(self, method, topic, envelope, context):

        identity = zpt.authorize(self.web_endpoint, envelope)

        if not identity:
            zpt.reply(envelope, {
                'status': self.unauthorized_status,
                'payload': {
                    'code': self.unauthorized_code,
                    'text': self.unauthorized_text
                }
            })
            return None

        return identity

    @staticmethod
    def default_callback(method, topic, reply, context):
        zpt.reply(context, reply)

    def path_discover(self, template_path, original_path, endpoint):

        template_path_parts = template_path.split('/')
        original_path_parts = original_path.split('/')

        if len(template_path_parts) != len(original_path_parts):
            raise Exception('The template topic and topic pattern received doesn\'t match.')

        groups = PARAM_PATTERN.findall(template_path)
        params = {}

        for group in groups:
            params[re.sub(r'[\{\}]', '', group)] = original_path_parts[template_path_parts.index(group)]

        return (endpoint.format(**params), params)

    def variable_resolver(self, data_template, variables):

        for key in data_template.keys():

            value = data_template[key]

            if type(value) is str:
                data_template[key] = value.format(**variables) if value else None

        return data_template if data_template else {}

    def dispatch(self, method, topic, envelope, context):

        identity = self.authorize(method, topic, envelope, context)

        if not identity:
            return # cancel the dispatch

        endpoint, variables = self.path_discover(self.web_endpoint, topic, self.data_layer_endpoint)

        zpt.route(
            method,
            endpoint,
            {
                'headers': zpt.auth_headers(identity),
                'params': zpt.merge(envelope.get('params'), self.variable_resolver(self.data_layer_params, variables)),
                'payload': envelope.get('payload')
            },
            {'context': envelope },
            getattr(self, '{}_callback'.format(method))
        )
<commit_msg>Adding extra data-layer params in payload and params dict<commit_after>import zpt
import re

METHODS = ['get', 'post', 'put', 'patch', 'delete', 'head']
PARAM_PATTERN = re.compile(r'(\{[^\}]+\})')

class RestHandler(object):

    def __init__(self):
        self.web_endpoint = None
        self.data_layer_endpoint = None
        self.data_layer_params = None
        self.methods_allowed = METHODS
        self.unauthorized_status = 401
        self.unauthorized_code = 1019
        self.unauthorized_text = 'unauthorized access to this resource'

    def __getattribute__(self, name):

        try:
            return object.__getattribute__(self, name)
        except Exception as e:

            if name in self.methods_allowed:
                return self.dispatch
            elif name in ['{}_callback'.format(method) for method in self.methods_allowed]:
                return self.default_callback

        super(RestHandler, self).__getattribute__(name)

    def __setattribute__(self, name, value):

        if name == 'methods_allowed':
            return [method for method in self.__methods_allowed if method in METHODS]

        super(RestHandler, self).__setattribute__(name, value)

    def authorize(self, method, topic, envelope, context):

        identity = zpt.authorize(self.web_endpoint, envelope)

        if not identity:
            zpt.reply(envelope, {
                'status': self.unauthorized_status,
                'payload': {
                    'code': self.unauthorized_code,
                    'text': self.unauthorized_text
                }
            })
            return None

        return identity

    @staticmethod
    def default_callback(method, topic, reply, context):
        zpt.reply(context, reply)

    def path_discover(self, template_path, original_path, endpoint):

        template_path_parts = template_path.split('/')
        original_path_parts = original_path.split('/')

        if len(template_path_parts) != len(original_path_parts):
            raise Exception('The template topic and topic pattern received doesn\'t match.')

        groups = PARAM_PATTERN.findall(template_path)
        params = {}

        for group in groups:
            params[re.sub(r'[\{\}]', '', group)] = original_path_parts[template_path_parts.index(group)]

        return (endpoint.format(**params), params)

    def variable_resolver(self, data_template, variables):

        for key in data_template.keys():

            value = data_template[key]

            if type(value) is str:
                data_template[key] = value.format(**variables) if value else None

        return data_template if data_template else {}

    def dispatch(self, method, topic, envelope, context):

        identity = self.authorize(method, topic, envelope, context)

        if not identity:
            return # cancel the dispatch

        endpoint, variables = self.path_discover(self.web_endpoint, topic, self.data_layer_endpoint)

        params = envelope.get('params')
        payload = envelope.get('payload')

        if params: params = zpt.merge(params, self.variable_resolver(self.data_layer_params, variables))
        if payload: payload = zpt.merge(payload, self.variable_resolver(self.data_layer_params, variables))

        zpt.route(
            method,
            endpoint,
            {
                'headers': zpt.auth_headers(identity),
                'params': params,
                'payload': payload
            },
            {'context': envelope },
            getattr(self, '{}_callback'.format(method))
        )

<|endoftext|>


<commit_before>"""
    You have a function rand7() that generates a random integer from 1 to 7.
    Use it to write a function rand5() that generates a random integer from 1 to 5.
    rand7() returns each integer with equal probability. rand5() must also return each integer with equal probability.
"""
<commit_msg>Feat: Completed problem 38<commit_after>"""
    You have a function rand7() that generates a random integer from 1 to 7.
    Use it to write a function rand5() that generates a random integer from 1 to 5.
    rand7() returns each integer with equal probability. rand5() must also return each integer with equal probability.
"""
import unittest
import random

def rand7():
    return random.randrange(1, 7)

def rand5():
    r = rand7()
    if r > 5:
        r = rand7()
    return r

class MyTestCases(unittest.TestCase):
    def test_rand5(self):
        pass
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

# Copyright (c) 2017 Jesse Pinnell
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

""" HTML rendering classes """

import abc
import textwrap

# pylint: disable=too-few-public-methods
# pylint: disable=no-self-use
# pylint: disable=unused-argument
# pylint: disable=no-member

class BaseRenderer(metaclass=abc.ABCMeta):
    """ Abstract base class for rendering classes """

    def render(self, exercise_data):
        """ Render the exercise data
        TODO this only handles a single set of data; needs to be more powerful (PoC)
        """
        return ""

class BasicHTMLRenderer(object):
    """ Renderer that creates a VERY basic html table of data """

    def render(self, title, exercise_data):
        """ Render the excercise data in HTML

        This is a simple table for printing onto paper (not meant to be pretty)
        """

        if len(exercise_data) == 1:
            body = "
\n"
            body += "
Exercise	Muscles	Fixture(s)	"\
                "Rig(s)
\n"
            this_exercise_data = exercise_data[0]
            for exercise in this_exercise_data:
                if exercise.rigs is not None:
                    rig_names = ", ".join(["({0})".format(rig.name) \
                       if rig.optional else rig.name for rig in exercise.rigs])
                else:
                    rig_names = ""
                body += "{0}
{1}	{2}	{3}	"\
                    "{4}
\n".format(\
                        " " * 16, exercise.name, ", ".join(exercise.muscles_exercised),\
                        ", ".join(exercise.fixtures), rig_names)
            body += "
\n"

            html = """
            <!DOCTYPE html>

                <meta charset="UTF-8">
                <title>{0}</title>

            {1}

            """.format(title, body)
        else:
            body = "
\n    "

            num_sets_per_exercise = 3
            for day_i, day in enumerate(exercise_data):
                body += "
Day {0}	".format(day_i + 1)
                for exercise in day:
                    body += "
{1}
{0}	".format(exercise.name, "	"\
                        * num_sets_per_exercise)
                body += "
\n"

            body += "
"
            html = """
            <!DOCTYPE html>

                <meta charset="UTF-8">
                <title>{0}</title>

            {1}

            """.format(title, body)

        return textwrap.dedent(html)

BaseRenderer.register(BasicHTMLRenderer)
<commit_msg>Add fixture color<commit_after>#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

# Copyright (c) 2017 Jesse Pinnell
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

""" HTML rendering classes """

import abc
import textwrap

# pylint: disable=too-few-public-methods
# pylint: disable=no-self-use
# pylint: disable=unused-argument
# pylint: disable=no-member

class BaseRenderer(metaclass=abc.ABCMeta):
    """ Abstract base class for rendering classes """

    def render(self, exercise_data):
        """ Render the exercise data
        TODO this only handles a single set of data; needs to be more powerful (PoC)
        """
        return ""

class BasicHTMLRenderer(object):
    """ Renderer that creates a VERY basic html table of data """

    def render(self, title, exercise_data):
        """ Render the excercise data in HTML

        This is a simple table for printing onto paper (not meant to be pretty)
        """

        if len(exercise_data) == 1:
            body = "
\n"
            body += "
Exercise	Muscles	Fixture(s)	"\
                "Rig(s)
\n"
            this_exercise_data = exercise_data[0]
            for exercise in this_exercise_data:
                if exercise.rigs is not None:
                    rig_names = ", ".join(["({0})".format(rig.name) \
                       if rig.optional else rig.name for rig in exercise.rigs])
                else:
                    rig_names = ""
                body += "{0}
{1}	{2}	{3}	"\
                    "{4}
\n".format(\
                        " " * 16, exercise.name, ", ".join(exercise.muscles_exercised),\
                        ", ".join(exercise.fixtures), rig_names)
            body += "
\n"

            html = """
            <!DOCTYPE html>

                <meta charset="UTF-8">
                <title>{0}</title>

            {1}

            """.format(title, body)
        else:
            body = "
\n    "

            num_sets_per_exercise = 3
            for day_i, day in enumerate(exercise_data):
                body += "
Day {0}	".format(day_i + 1)
                for exercise in day:
                    if exercise.fixtures:
                        fixture_class = list(exercise.fixtures)[0].replace(' ', '_').replace(',', '')
                    checkboxes = "	" * num_sets_per_exercise
                    body += f"
{checkboxes}
"
                    body += f"

"
                    body += f"{exercise.name}	\n"
                body += "
\n"

            body += "
"
            html = """
            <!DOCTYPE html>

                <meta charset="UTF-8">
                <title>{0}</title>

            {1}

            """.format(title, body)

        return textwrap.dedent(html)

BaseRenderer.register(BasicHTMLRenderer)
<|endoftext|>


<commit_before>from .base import *
from .input_checks import *

class LogisticBalance():
    """
    A special case where the handicap functions are logistic functions
    whose argument is row_handicap - col_handicap + offset,
    where offset is chosen so that when all h are zero the base_matrix is recovered.
    Commonly payoffs represent win rates.

    The optimization is done over the canonical range (-0.5, 0.5).
    The input is shifted from (0, max_payoff) to this range, and then decanonicalized at the end.
    """

    def __init__(self, base_matrix):
        """
        Sets self.base_matrix.
        Args:
            base_matrix
        Raises:
            ValueError:
                If any element of base_matrix is not in the open interval (0, max_payoff).
            ValueWarning:
                If base_matrix is not (close to) skew-symmetric plus a constant offset.
                If base_matrix has elements close to 0 and/or max_payoff.
        """

        # Check bounds.
        if numpy.any(base_matrix <= 0.0) or numpy.any(base_matrix >= self.max_payoff):
            raise ValueError('base_matrix has element(s) not in the open interval (0, max_payoff), where max_payoff = %f is twice the value of the game.' % self.max_payoff)
        if numpy.any(numpy.isclose(base_matrix, 0.0)) or numpy.any(numpy.isclose(base_matrix, self.max_payoff)):
            warnings.warn('base_matrix has element(s) close to 0 and/or max_payoff, where max_payoff = %f is twice the value of the game.' % self.max_payoff, ValueWarning)

        self.base_matrix = base_matrix
        self.initial_offset_matrix = numpy.log(self.max_payoff / base_matrix - 1.0)

        check_shape(self.base_matrix, self.row_weights, self.col_weights)

    def handicap_function(self, h_r, h_c):
        return 1.0 / (1.0 + numpy.exp(h_r[:, None] - h_c[None, :] + self.initial_offset_matrix)) - 0.5

    def row_derivative(self, h_r, h_c):
        payoffs = self.handicap_function(h_r, h_c)
        return payoffs * payoffs - 0.25

    def col_derivative(self, h_r, h_c):
        payoffs = self.handicap_function(h_r, h_c)
        return 0.25 - payoffs * payoffs

    def decanonicalize_F(self, F):
        """ Expands the payoff matrix back to the original range (0, max_payoff). """
        payoff_matrix = (F + 0.5) * self.max_payoff
        return payoff_matrix

class LogisticNonSymmetricBalance(LogisticBalance, NonSymmetricBalance):
    def __init__(self, base_matrix, value, max_payoff, row_weights = None, col_weights = None):
        """
        Args:
            base_matrix: The elements of base_matrix must be in (0, max_payoff),
                where max_payoff is twice the value of the game.
                The base_matrix should be skew-symmetric plus a constant offset (namely the value of the game).
                In particular, all diagonal elements should be equal to the value of the game.
            row_weights, col_weights: Defines the desired Nash equilibrium in terms of strategy probability weights.
                If only an integer is specified, a uniform distribution will be used.
        Raises:
            ValueError:
                If any element of base_matrix is not in the open interval (0, max_payoff).
                If the size of strategy_weights does not match the dimensions of base_matrix.
            ValueWarning:
                If base_matrix is not (close to) skew-symmetric plus a constant offset.
                If base_matrix has elements close to 0 and/or max_payoff.
                    max_payoff is twice the value of the game.
        """
        if row_weights is None: row_weights = base_matrix.shape[0]
        if col_weights is None: col_weights = base_matrix.shape[1]

        self.max_payoff = max_payoff
        normalized_value = value / max_payoff - 0.5

        NonSymmetricBalance.__init__(self, row_weights, col_weights, value = normalized_value)
        LogisticBalance.__init__(self, base_matrix)

class LogisticSymmetricBalance(LogisticBalance, SymmetricBalance):
    """
    Symmetric verison.
    """
    def __init__(self, base_matrix, strategy_weights = None):
        """
        Args:
            base_matrix: The elements of base_matrix must be in (0, max_payoff),
                where max_payoff is twice the value of the game.
                The base_matrix should be skew-symmetric plus a constant offset (namely the value of the game).
                In particular, all diagonal elements should be equal to the value of the game.
            strategy_weights: Defines the desired Nash equilibrium in terms of strategy probability weights.
                If only an integer is specified, a uniform distribution will be used.
        Raises:
            ValueError:
                If base_matrix is not square.
                If any element of base_matrix is not in the open interval (0, max_payoff).
                If the size of strategy_weights does not match the dimensions of base_matrix.
            ValueWarning:
                If base_matrix is not (close to) skew-symmetric plus a constant offset.
                If base_matrix has elements close to 0 and/or max_payoff.
                    max_payoff is twice the value of the game.
        """
        if strategy_weights is None: strategy_weights = base_matrix.shape[0]

        value = base_matrix[0, 0]

        check_square(base_matrix)
        check_skew_symmetry(base_matrix, value)

        # The maximum possible payoff (e.g. 100% win rate) is twice the value of the game.
        self.max_payoff = 2.0 * value

        SymmetricBalance.__init__(self, strategy_weights)
        LogisticBalance.__init__(self, base_matrix)
<commit_msg>tweak LogisticNonSymmetricBalance args<commit_after>from .base import *
from .input_checks import *

class LogisticBalance():
    """
    A special case where the handicap functions are logistic functions
    whose argument is row_handicap - col_handicap + offset,
    where offset is chosen so that when all h are zero the base_matrix is recovered.
    Commonly payoffs represent win rates.

    The optimization is done over the canonical range (-0.5, 0.5).
    The input is shifted from (0, max_payoff) to this range, and then decanonicalized at the end.
    """

    def __init__(self, base_matrix):
        """
        Sets self.base_matrix.
        Args:
            base_matrix
        Raises:
            ValueError:
                If any element of base_matrix is not in the open interval (0, max_payoff).
            ValueWarning:
                If base_matrix is not (close to) skew-symmetric plus a constant offset.
                If base_matrix has elements close to 0 and/or max_payoff.
        """

        # Check bounds.
        if numpy.any(base_matrix <= 0.0) or numpy.any(base_matrix >= self.max_payoff):
            raise ValueError('base_matrix has element(s) not in the open interval (0, max_payoff), where max_payoff = %f is twice the value of the game.' % self.max_payoff)
        if numpy.any(numpy.isclose(base_matrix, 0.0)) or numpy.any(numpy.isclose(base_matrix, self.max_payoff)):
            warnings.warn('base_matrix has element(s) close to 0 and/or max_payoff, where max_payoff = %f is twice the value of the game.' % self.max_payoff, ValueWarning)

        self.base_matrix = base_matrix
        self.initial_offset_matrix = numpy.log(self.max_payoff / base_matrix - 1.0)

        check_shape(self.base_matrix, self.row_weights, self.col_weights)

    def handicap_function(self, h_r, h_c):
        return 1.0 / (1.0 + numpy.exp(h_r[:, None] - h_c[None, :] + self.initial_offset_matrix)) - 0.5

    def row_derivative(self, h_r, h_c):
        payoffs = self.handicap_function(h_r, h_c)
        return payoffs * payoffs - 0.25

    def col_derivative(self, h_r, h_c):
        payoffs = self.handicap_function(h_r, h_c)
        return 0.25 - payoffs * payoffs

    def decanonicalize_F(self, F):
        """ Expands the payoff matrix back to the original range (0, max_payoff). """
        payoff_matrix = (F + 0.5) * self.max_payoff
        return payoff_matrix

class LogisticNonSymmetricBalance(LogisticBalance, NonSymmetricBalance):
    def __init__(self, base_matrix, max_payoff, row_weights = None, col_weights = None, value = None):
        """
        Args:
            base_matrix: The elements of base_matrix must be in (0, max_payoff).
            max_payoff: Maximum payoff of the game. All b
            row_weights, col_weights: Defines the desired Nash equilibrium in terms of strategy probability weights.
                If only an integer is specified, a uniform distribution will be used.
            value: Desired value of the game. Defaults to 0.5 * max_payoff.
        Raises:
            ValueError:
                If any element of base_matrix is not in the open interval (0, max_payoff).
                If the size of strategy_weights does not match the dimensions of base_matrix.
            ValueWarning:
                If base_matrix is not (close to) skew-symmetric plus a constant offset.
                If base_matrix has elements close to 0 and/or max_payoff.
                    max_payoff is twice the value of the game.
        """
        if row_weights is None: row_weights = base_matrix.shape[0]
        if col_weights is None: col_weights = base_matrix.shape[1]

        self.max_payoff = max_payoff
        if value is None: value = 0.5 * max_payoff

        if value <= 0.0 or value >= max_payoff:
            raise ValueError("value is not in the range (0, max_payoff).")

        if numpy.any(base_matrix <= 0.0) or numpy.any(base_matrix >= max_payoff):
            raise ValueError("base_matrix contains element(s) not in the range (0, max_payoff).")

        normalized_value = value / max_payoff - 0.5

        NonSymmetricBalance.__init__(self, row_weights, col_weights, value = normalized_value)
        LogisticBalance.__init__(self, base_matrix)

class LogisticSymmetricBalance(LogisticBalance, SymmetricBalance):
    """
    Symmetric verison.
    """
    def __init__(self, base_matrix, strategy_weights = None):
        """
        Args:
            base_matrix: The elements of base_matrix must be in (0, max_payoff),
                where max_payoff is twice the value of the game.
                The base_matrix should be skew-symmetric plus a constant offset (namely the value of the game).
                In particular, all diagonal elements should be equal to the value of the game.
            strategy_weights: Defines the desired Nash equilibrium in terms of strategy probability weights.
                If only an integer is specified, a uniform distribution will be used.
        Raises:
            ValueError:
                If base_matrix is not square.
                If any element of base_matrix is not in the open interval (0, max_payoff).
                If the size of strategy_weights does not match the dimensions of base_matrix.
            ValueWarning:
                If base_matrix is not (close to) skew-symmetric plus a constant offset.
                If base_matrix has elements close to 0 and/or max_payoff.
                    max_payoff is twice the value of the game.
        """
        if strategy_weights is None: strategy_weights = base_matrix.shape[0]

        value = base_matrix[0, 0]

        check_square(base_matrix)
        check_skew_symmetry(base_matrix, value)

        # The maximum possible payoff (e.g. 100% win rate) is twice the value of the game.
        self.max_payoff = 2.0 * value

        SymmetricBalance.__init__(self, strategy_weights)
        LogisticBalance.__init__(self, base_matrix)
<|endoftext|>


<commit_before>"""
Utilities for working with the local dataset cache.
This file is adapted from the AllenNLP library at https://github.com/allenai/allennlp
Copyright by the AllenNLP authors.
"""
from __future__ import (absolute_import, division, print_function, unicode_literals)

import sys
import json
import logging
import os
import shutil
import tempfile
import fnmatch
from functools import wraps
from hashlib import sha256
import sys
from io import open

import boto3
import requests
from botocore.exceptions import ClientError
from tqdm import tqdm

try:
    from torch.hub import _get_torch_home
    torch_cache_home = _get_torch_home()
except ImportError:
    torch_cache_home = os.path.expanduser(
        os.getenv('TORCH_HOME', os.path.join(
            os.getenv('XDG_CACHE_HOME', '~/.cache'), 'torch')))
default_cache_path = os.path.join(torch_cache_home, 'pytorch_transformers')

try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse

try:
    from pathlib import Path
    PYTORCH_PRETRAINED_BERT_CACHE = Path(
        os.getenv('PYTORCH_PRETRAINED_BERT_CACHE', default_cache_path))
except (AttributeError, ImportError):
    PYTORCH_PRETRAINED_BERT_CACHE = os.getenv('PYTORCH_PRETRAINED_BERT_CACHE',
                                              default_cache_path)

logger = logging.getLogger(__name__)  # pylint: disable=invalid-name

def url_to_filename(url, etag=None):
    """
    Convert `url` into a hashed filename in a repeatable way.
    If `etag` is specified, append its hash to the url's, delimited
    by a period.
    """
    url_bytes = url.encode('utf-8')
    url_hash = sha256(url_bytes)
    filename = url_hash.hexdigest()

    if etag:
        etag_bytes = etag.encode('utf-8')
        etag_hash = sha256(etag_bytes)
        filename += '.' + etag_hash.hexdigest()

    return filename

def filename_to_url(filename, cache_dir=None):
    """
    Return the url and etag (which may be ``None``) stored for `filename`.
    Raise ``EnvironmentError`` if `filename` or its stored metadata do not exist.
    """
    if cache_dir is None:
        cache_dir = PYTORCH_PRETRAINED_BERT_CACHE
    if sys.version_info[0] == 3 and isinstance(cache_dir, Path):
        cache_dir = str(cache_dir)

    cache_path = os.path.join(cache_dir, filename)
    if not os.path.exists(cache_path):
        raise EnvironmentError("file {} not found".format(cache_path))

    meta_path = cache_path + '.json'
    if not os.path.exists(meta_path):
        raise EnvironmentError("file {} not found".format(meta_path))

    with open(meta_path, encoding="utf-8") as meta_file:
        metadata = json.load(meta_file)
    url = metadata['url']
    etag = metadata['etag']

    return url, etag

def cached_path(url_or_filename, cache_dir=None):
    """
    Given something that might be a URL (or might be a local path),
    determine which. If it's a URL, download the file and cache it, and
    return the path to the cached file. If it's already a local path,
    make sure the file exists and then return the path.
    """
    if cache_dir is None:
        cache_dir = PYTORCH_PRETRAINED_BERT_CACHE
    if sys.version_info[0] == 3 and isinstance(url_or_filename, Path):
        url_or_filename = str(url_or_filename)
    if sys.version_info[0] == 3 and isinstance(cache_dir, Path):
        cache_dir = str(cache_dir)

    parsed = urlparse(url_or_filename)

    if parsed.scheme in ('http', 'https', 's3'):
        # URL, so get it from the cache (downloading if necessary)
        return get_from_cache(url_or_filename, cache_dir)
    elif os.path.exists(url_or_filename):
        # File, and it exists.
        return url_or_filename
    elif parsed.scheme == '':
        # File, but it doesn't exist.
        raise EnvironmentError("file {} not found".format(url_or_filename))
    else:
        # Something unknown
        raise ValueError("unable to parse {} as a URL or as a local path".format(url_or_filename))

def split_s3_path(url):
    """Split a full s3 path into the bucket name and path."""
    parsed = urlparse(url)
    if not parsed.netloc or not parsed.path:
        raise ValueError("bad s3 path {}".format(url))
    bucket_name = parsed.netloc
    s3_path = parsed.path
    # Remove '/' at beginning of path.
    if s3_path.startswith("/"):
        s3_path = s3_path[1:]
    return bucket_name, s3_path

def s3_request(func):
    """
    Wrapper function for s3 requests in order to create more helpful error
    messages.
    """

    @wraps(func)
    def wrapper(url, *args, **kwargs):
        try:
            return func(url, *args, **kwargs)
        except ClientError as exc:
            if int(exc.response["Error"]["Code"]) == 404:
                raise EnvironmentError("file {} not found".format(url))
            else:
                raise

    return wrapper

@s3_request
def s3_etag(url):
    """Check ETag on S3 object."""
    s3_resource = boto3.resource("s3")
    bucket_name, s3_path = split_s3_path(url)
    s3_object = s3_resource.Object(bucket_name, s3_path)
    return s3_object.e_tag

@s3_request
def s3_get(url, temp_file):
    """Pull a file directly from S3."""
    s3_resource = boto3.resource("s3")
    bucket_name, s3_path = split_s3_path(url)
    s3_resource.Bucket(bucket_name).download_fileobj(s3_path, temp_file)

def http_get(url, temp_file):
    req = requests.get(url, stream=True)
    content_length = req.headers.get('Content-Length')
    total = int(content_length) if content_length is not None else None
    progress = tqdm(unit="B", total=total)
    for chunk in req.iter_content(chunk_size=1024):
        if chunk: # filter out keep-alive new chunks
            progress.update(len(chunk))
            temp_file.write(chunk)
    progress.close()

def get_from_cache(url, cache_dir=None):
    """
    Given a URL, look for the corresponding dataset in the local cache.
    If it's not there, download it. Then return the path to the cached file.
    """
    if cache_dir is None:
        cache_dir = PYTORCH_PRETRAINED_BERT_CACHE
    if sys.version_info[0] == 3 and isinstance(cache_dir, Path):
        cache_dir = str(cache_dir)
    if sys.version_info[0] == 2 and not isinstance(cache_dir, str):
        cache_dir = str(cache_dir)

    if not os.path.exists(cache_dir):
        os.makedirs(cache_dir)

    # Get eTag to add to filename, if it exists.
    if url.startswith("s3://"):
        etag = s3_etag(url)
    else:
        try:
            response = requests.head(url, allow_redirects=True)
            if response.status_code != 200:
                etag = None
            else:
                etag = response.headers.get("ETag")
        except EnvironmentError:
            etag = None

    if sys.version_info[0] == 2 and etag is not None:
        etag = etag.decode('utf-8')
    filename = url_to_filename(url, etag)

    # get cache path to put the file
    cache_path = os.path.join(cache_dir, filename)

    # If we don't have a connection (etag is None) and can't identify the file
    # try to get the last downloaded one
    if not os.path.exists(cache_path) and etag is None:
        matching_files = fnmatch.filter(os.listdir(cache_dir), filename + '.*')
        matching_files = list(filter(lambda s: not s.endswith('.json'), matching_files))
        if matching_files:
            cache_path = os.path.join(cache_dir, matching_files[-1])

    if not os.path.exists(cache_path):
        # Download to temporary file, then copy to cache dir once finished.
        # Otherwise you get corrupt cache entries if the download gets interrupted.
        with tempfile.NamedTemporaryFile() as temp_file:
            logger.info("%s not found in cache, downloading to %s", url, temp_file.name)

            # GET file object
            if url.startswith("s3://"):
                s3_get(url, temp_file)
            else:
                http_get(url, temp_file)

            # we are copying the file before closing it, so flush to avoid truncation
            temp_file.flush()
            # shutil.copyfileobj() starts at the current position, so go to the start
            temp_file.seek(0)

            logger.info("copying %s to cache at %s", temp_file.name, cache_path)
            with open(cache_path, 'wb') as cache_file:
                shutil.copyfileobj(temp_file, cache_file)

            logger.info("creating metadata file for %s", cache_path)
            meta = {'url': url, 'etag': etag}
            meta_path = cache_path + '.json'
            with open(meta_path, 'w') as meta_file:
                output_string = json.dumps(meta)
                if sys.version_info[0] == 2 and isinstance(output_string, str):
                    output_string = unicode(output_string, 'utf-8')  # The beauty of python 2
                meta_file.write(output_string)

            logger.info("removing temp file %s", temp_file.name)

    return cache_path
<commit_msg>import sys twice<commit_after>"""
Utilities for working with the local dataset cache.
This file is adapted from the AllenNLP library at https://github.com/allenai/allennlp
Copyright by the AllenNLP authors.
"""
from __future__ import (absolute_import, division, print_function, unicode_literals)

import sys
import json
import logging
import os
import shutil
import tempfile
import fnmatch
from functools import wraps
from hashlib import sha256
from io import open

import boto3
import requests
from botocore.exceptions import ClientError
from tqdm import tqdm

try:
    from torch.hub import _get_torch_home
    torch_cache_home = _get_torch_home()
except ImportError:
    torch_cache_home = os.path.expanduser(
        os.getenv('TORCH_HOME', os.path.join(
            os.getenv('XDG_CACHE_HOME', '~/.cache'), 'torch')))
default_cache_path = os.path.join(torch_cache_home, 'pytorch_transformers')

try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse

try:
    from pathlib import Path
    PYTORCH_PRETRAINED_BERT_CACHE = Path(
        os.getenv('PYTORCH_PRETRAINED_BERT_CACHE', default_cache_path))
except (AttributeError, ImportError):
    PYTORCH_PRETRAINED_BERT_CACHE = os.getenv('PYTORCH_PRETRAINED_BERT_CACHE',
                                              default_cache_path)

logger = logging.getLogger(__name__)  # pylint: disable=invalid-name

def url_to_filename(url, etag=None):
    """
    Convert `url` into a hashed filename in a repeatable way.
    If `etag` is specified, append its hash to the url's, delimited
    by a period.
    """
    url_bytes = url.encode('utf-8')
    url_hash = sha256(url_bytes)
    filename = url_hash.hexdigest()

    if etag:
        etag_bytes = etag.encode('utf-8')
        etag_hash = sha256(etag_bytes)
        filename += '.' + etag_hash.hexdigest()

    return filename

def filename_to_url(filename, cache_dir=None):
    """
    Return the url and etag (which may be ``None``) stored for `filename`.
    Raise ``EnvironmentError`` if `filename` or its stored metadata do not exist.
    """
    if cache_dir is None:
        cache_dir = PYTORCH_PRETRAINED_BERT_CACHE
    if sys.version_info[0] == 3 and isinstance(cache_dir, Path):
        cache_dir = str(cache_dir)

    cache_path = os.path.join(cache_dir, filename)
    if not os.path.exists(cache_path):
        raise EnvironmentError("file {} not found".format(cache_path))

    meta_path = cache_path + '.json'
    if not os.path.exists(meta_path):
        raise EnvironmentError("file {} not found".format(meta_path))

    with open(meta_path, encoding="utf-8") as meta_file:
        metadata = json.load(meta_file)
    url = metadata['url']
    etag = metadata['etag']

    return url, etag

def cached_path(url_or_filename, cache_dir=None):
    """
    Given something that might be a URL (or might be a local path),
    determine which. If it's a URL, download the file and cache it, and
    return the path to the cached file. If it's already a local path,
    make sure the file exists and then return the path.
    """
    if cache_dir is None:
        cache_dir = PYTORCH_PRETRAINED_BERT_CACHE
    if sys.version_info[0] == 3 and isinstance(url_or_filename, Path):
        url_or_filename = str(url_or_filename)
    if sys.version_info[0] == 3 and isinstance(cache_dir, Path):
        cache_dir = str(cache_dir)

    parsed = urlparse(url_or_filename)

    if parsed.scheme in ('http', 'https', 's3'):
        # URL, so get it from the cache (downloading if necessary)
        return get_from_cache(url_or_filename, cache_dir)
    elif os.path.exists(url_or_filename):
        # File, and it exists.
        return url_or_filename
    elif parsed.scheme == '':
        # File, but it doesn't exist.
        raise EnvironmentError("file {} not found".format(url_or_filename))
    else:
        # Something unknown
        raise ValueError("unable to parse {} as a URL or as a local path".format(url_or_filename))

def split_s3_path(url):
    """Split a full s3 path into the bucket name and path."""
    parsed = urlparse(url)
    if not parsed.netloc or not parsed.path:
        raise ValueError("bad s3 path {}".format(url))
    bucket_name = parsed.netloc
    s3_path = parsed.path
    # Remove '/' at beginning of path.
    if s3_path.startswith("/"):
        s3_path = s3_path[1:]
    return bucket_name, s3_path

def s3_request(func):
    """
    Wrapper function for s3 requests in order to create more helpful error
    messages.
    """

    @wraps(func)
    def wrapper(url, *args, **kwargs):
        try:
            return func(url, *args, **kwargs)
        except ClientError as exc:
            if int(exc.response["Error"]["Code"]) == 404:
                raise EnvironmentError("file {} not found".format(url))
            else:
                raise

    return wrapper

@s3_request
def s3_etag(url):
    """Check ETag on S3 object."""
    s3_resource = boto3.resource("s3")
    bucket_name, s3_path = split_s3_path(url)
    s3_object = s3_resource.Object(bucket_name, s3_path)
    return s3_object.e_tag

@s3_request
def s3_get(url, temp_file):
    """Pull a file directly from S3."""
    s3_resource = boto3.resource("s3")
    bucket_name, s3_path = split_s3_path(url)
    s3_resource.Bucket(bucket_name).download_fileobj(s3_path, temp_file)

def http_get(url, temp_file):
    req = requests.get(url, stream=True)
    content_length = req.headers.get('Content-Length')
    total = int(content_length) if content_length is not None else None
    progress = tqdm(unit="B", total=total)
    for chunk in req.iter_content(chunk_size=1024):
        if chunk: # filter out keep-alive new chunks
            progress.update(len(chunk))
            temp_file.write(chunk)
    progress.close()

def get_from_cache(url, cache_dir=None):
    """
    Given a URL, look for the corresponding dataset in the local cache.
    If it's not there, download it. Then return the path to the cached file.
    """
    if cache_dir is None:
        cache_dir = PYTORCH_PRETRAINED_BERT_CACHE
    if sys.version_info[0] == 3 and isinstance(cache_dir, Path):
        cache_dir = str(cache_dir)
    if sys.version_info[0] == 2 and not isinstance(cache_dir, str):
        cache_dir = str(cache_dir)

    if not os.path.exists(cache_dir):
        os.makedirs(cache_dir)

    # Get eTag to add to filename, if it exists.
    if url.startswith("s3://"):
        etag = s3_etag(url)
    else:
        try:
            response = requests.head(url, allow_redirects=True)
            if response.status_code != 200:
                etag = None
            else:
                etag = response.headers.get("ETag")
        except EnvironmentError:
            etag = None

    if sys.version_info[0] == 2 and etag is not None:
        etag = etag.decode('utf-8')
    filename = url_to_filename(url, etag)

    # get cache path to put the file
    cache_path = os.path.join(cache_dir, filename)

    # If we don't have a connection (etag is None) and can't identify the file
    # try to get the last downloaded one
    if not os.path.exists(cache_path) and etag is None:
        matching_files = fnmatch.filter(os.listdir(cache_dir), filename + '.*')
        matching_files = list(filter(lambda s: not s.endswith('.json'), matching_files))
        if matching_files:
            cache_path = os.path.join(cache_dir, matching_files[-1])

    if not os.path.exists(cache_path):
        # Download to temporary file, then copy to cache dir once finished.
        # Otherwise you get corrupt cache entries if the download gets interrupted.
        with tempfile.NamedTemporaryFile() as temp_file:
            logger.info("%s not found in cache, downloading to %s", url, temp_file.name)

            # GET file object
            if url.startswith("s3://"):
                s3_get(url, temp_file)
            else:
                http_get(url, temp_file)

            # we are copying the file before closing it, so flush to avoid truncation
            temp_file.flush()
            # shutil.copyfileobj() starts at the current position, so go to the start
            temp_file.seek(0)

            logger.info("copying %s to cache at %s", temp_file.name, cache_path)
            with open(cache_path, 'wb') as cache_file:
                shutil.copyfileobj(temp_file, cache_file)

            logger.info("creating metadata file for %s", cache_path)
            meta = {'url': url, 'etag': etag}
            meta_path = cache_path + '.json'
            with open(meta_path, 'w') as meta_file:
                output_string = json.dumps(meta)
                if sys.version_info[0] == 2 and isinstance(output_string, str):
                    output_string = unicode(output_string, 'utf-8')  # The beauty of python 2
                meta_file.write(output_string)

            logger.info("removing temp file %s", temp_file.name)

    return cache_path
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
:Author: Akshay Paropkari

:Contact: paropkari.1@gmail.com

:Date Created: 10/15/2014

:Abstract: Automated Tests for BIOM calculation.
"""
import unittest
import json
import qiime_tools
from qiime_tools import biom_calc as bc
import math

class biom_calc_Test(unittest.TestCase):

    def setUp(self):
        """
        Initializing BIOM format file.
        """
        self.biom_text = """{
                     "id":null,
                     "format": "Biological Observation Matrix 0.9.1-dev",
                     "format_url": "http://biom-format.org/documentation/format_versions/biom-1.0.html",
                     "type": "OTU table",
                     "generated_by": "QIIME revision 1.4.0-dev",
                     "date": "2011-12-19T19:00:00",
                     "rows":[
                        {"id":"GG_OTU_1", "metadata":{"taxonomy":["k__Bacteria", "p__Proteobacteria", "c__Gammaproteobacteria", "o__Enterobacteriales", "f__Enterobacteriaceae", "g__Escherichia", "s__"]}},
                        {"id":"GG_OTU_2", "metadata":{"taxonomy":["k__Bacteria", "p__Cyanobacteria", "c__Nostocophycideae", "o__Nostocales", "f__Nostocaceae", "g__Dolichospermum", "s__"]}},
                        {"id":"GG_OTU_3", "metadata":{"taxonomy":["k__Archaea", "p__Euryarchaeota", "c__Methanomicrobia", "o__Methanosarcinales", "f__Methanosarcinaceae", "g__Methanosarcina", "s__"]}},
                        {"id":"GG_OTU_4", "metadata":{"taxonomy":["k__Bacteria", "p__Firmicutes", "c__Clostridia", "o__Halanaerobiales", "f__Halanaerobiaceae", "g__Halanaerobium", "s__Halanaerobiumsaccharolyticum"]}},
                        {"id":"GG_OTU_5", "metadata":{"taxonomy":["k__Bacteria", "p__Proteobacteria", "c__Gammaproteobacteria", "o__Enterobacteriales", "f__Enterobacteriaceae", "g__Escherichia", "s__"]}}
                        ],
                     "columns":[
                        {"id":"Sample1", "metadata":{
                                                 "BarcodeSequence":"CGCTTATCGAGA",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"gut",
                                                 "Description":"human gut"}},
                        {"id":"Sample2", "metadata":{
                                                 "BarcodeSequence":"CATACCAGTAGC",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"gut",
                                                 "Description":"human gut"}},
                        {"id":"Sample3", "metadata":{
                                                 "BarcodeSequence":"CTCTCTACCTGT",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"gut",
                                                 "Description":"human gut"}},
                        {"id":"Sample4", "metadata":{
                                                 "BarcodeSequence":"CTCTCGGCCTGT",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"skin",
                                                 "Description":"human skin"}},
                        {"id":"Sample5", "metadata":{
                                                 "BarcodeSequence":"CTCTCTACCAAT",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"skin",
                                                 "Description":"human skin"}},
                        {"id":"Sample6", "metadata":{
                                                 "BarcodeSequence":"CTAACTACCAAT",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"skin",
                                                 "Description":"human skin"}}
                                ],
                     "matrix_type": "sparse",
                     "matrix_element_type": "int",
                     "shape": [5, 6],
                     "data":[[0,2,1],
                             [1,0,5],
                             [1,1,1],
                             [1,3,2],
                             [1,4,3],
                             [1,5,1],
                             [2,2,1],
                             [2,3,4],
                             [2,5,2],
                             [3,0,2],
                             [3,1,1],
                             [3,2,1],
                             [3,5,1],
                             [4,1,1],
                             [4,2,1]
                            ]
                        }"""

        self.biom = json.loads(self.biom_text)

    def test_relative_abundance(self):
        """
        Testing relative abundance() function of biom.calc.py.

        :return: Returns OK, if testing goal is achieved, otherwise raises
                error.
        """
        sample = 'Sample3'
        self.result = bc.relative_abundance(self.biom, sample)

        # List containing manual calculations
        hand_calc = [1/4.0, 1/4.0, 1/4.0, 1/4.0]

        # Obtaining list of function calculated relative abundance for sample
        result1 = self.result.values()          # result1 is a list
        result2 = result1[0]                    # result2 is a dict
        func_calc = result2.values()            # list containing the calculated relative
                                                # abundance values

        # Testing the validity of relative_abundance() function.
        for hand, res in zip(hand_calc, func_calc):
            self.assertAlmostEqual(hand, res, msg='Relative abundances not calculated accurately.')

    def test_mean_otu_pct_abundance(self):
        """
        Testing mean_otu_pct_abundance() function of biom_calc.py.

        :return: Returns OK, if testing goal was achieved, otherwise raises
                error.
        """
        self.rel_a = bc.relative_abundance(self.biom,['Sample4'])
        self.result = bc.mean_otu_pct_abundance(self.rel_a, ['GG_OTU_2', 'GG_OTU_3'])

        # Obtaining lists of function calculations and manual hand calculations.
        func_calc = self.result.values()
        result1 = self.rel_a.values()            # result1 is a list
        result2 = result1[0]                     # result2 is a dict
        hand_calc = result2.values()             # list containing hand calculated
                                                 # relative abundance values

        # Testing the validity of the calculations of mean_otu_pct_abundance().
        for hand, res in zip(hand_calc, func_calc):
            self.assertAlmostEqual(hand*100, res, msg='Mean OTU not calculated accurately.')

    def test_MRA(self):
        """
        Testing mean relative abundance calculation, MRA() function of biom_calc.py.

        :return: Returns OK, if testing goal was achieved, otherwise raises error.
        """
        self.result = bc.MRA(self.biom, 'Sample4')
        self.mean_otu = bc.mean_otu_pct_abundance(
                bc.relative_abundance(self.biom,['Sample4']),
                ['GG_OTU_1','GG_OTU_2', 'GG_OTU_3','GG_OUTU4','GG_OTU_5']
                )

        # Obtaining lists of function calculations and manual hand calculations.
        func_calc = self.result.values()
        hand_calc = self.mean_otu.values()      # list containing hand calculated values

        # Testing the validity of the calculations of mean_otu_pct_abundance().
        for hand, res in zip(hand_calc, func_calc):
            self.assertAlmostEqual(hand, res, msg='Mean OTU not calculated accurately.')

    def test_raw_abundance(self):
        """
        Testing raw_abundance() function of biom_calc.py.

        :return: Returns OK, if testing goal is achieved, otherwise raises error.
        """
        self.result = {}
        self.result = bc.raw_abundance(self.biom, ['Sample1', 'Sample3'])

        # Lists containing hand and function calculated values.
        hand_calc = [
            math.log(1,10), math.log(5,10), math.log(1,10),
            math.log(3,10), math.log(1,10)
            ]
        func_calc = self.result.values()

        # Testing validity of raw_abundance() function.
        for hand, res in zip(hand_calc, func_calc):
            self.assertAlmostEqual(hand, res, msg='Raw abundances not calculated accurately!')

    def tearDown(self):
        """
        No particular event to clean, delete or close in testing biom_calc.py.
        """
        pass

if __name__ == '__main__':
    unittest.main()<commit_msg>Updated test_biom_calc.py<commit_after>#!/usr/bin/env python
"""
:Author: Akshay Paropkari

:Contact: paropkari.1@gmail.com

:Date Created: 10/15/2014

:Abstract: Automated Tests for BIOM calculation.
"""
import unittest
import json
import qiime_tools
from qiime_tools import biom_calc as bc
import math

class biom_calc_Test(unittest.TestCase):

    def setUp(self):
        """
        Initializing BIOM format file.
        """
        self.biom_text = """{
                     "id":null,
                     "format": "Biological Observation Matrix 0.9.1-dev",
                     "format_url": "http://biom-format.org/documentation/format_versions/biom-1.0.html",
                     "type": "OTU table",
                     "generated_by": "QIIME revision 1.4.0-dev",
                     "date": "2011-12-19T19:00:00",
                     "rows":[
                        {"id":"GG_OTU_1", "metadata":{"taxonomy":["k__Bacteria", "p__Proteobacteria", "c__Gammaproteobacteria", "o__Enterobacteriales", "f__Enterobacteriaceae", "g__Escherichia", "s__"]}},
                        {"id":"GG_OTU_2", "metadata":{"taxonomy":["k__Bacteria", "p__Cyanobacteria", "c__Nostocophycideae", "o__Nostocales", "f__Nostocaceae", "g__Dolichospermum", "s__"]}},
                        {"id":"GG_OTU_3", "metadata":{"taxonomy":["k__Archaea", "p__Euryarchaeota", "c__Methanomicrobia", "o__Methanosarcinales", "f__Methanosarcinaceae", "g__Methanosarcina", "s__"]}},
                        {"id":"GG_OTU_4", "metadata":{"taxonomy":["k__Bacteria", "p__Firmicutes", "c__Clostridia", "o__Halanaerobiales", "f__Halanaerobiaceae", "g__Halanaerobium", "s__Halanaerobiumsaccharolyticum"]}},
                        {"id":"GG_OTU_5", "metadata":{"taxonomy":["k__Bacteria", "p__Proteobacteria", "c__Gammaproteobacteria", "o__Enterobacteriales", "f__Enterobacteriaceae", "g__Escherichia", "s__"]}}
                        ],
                     "columns":[
                        {"id":"Sample1", "metadata":{
                                                 "BarcodeSequence":"CGCTTATCGAGA",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"gut",
                                                 "Description":"human gut"}},
                        {"id":"Sample2", "metadata":{
                                                 "BarcodeSequence":"CATACCAGTAGC",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"gut",
                                                 "Description":"human gut"}},
                        {"id":"Sample3", "metadata":{
                                                 "BarcodeSequence":"CTCTCTACCTGT",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"gut",
                                                 "Description":"human gut"}},
                        {"id":"Sample4", "metadata":{
                                                 "BarcodeSequence":"CTCTCGGCCTGT",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"skin",
                                                 "Description":"human skin"}},
                        {"id":"Sample5", "metadata":{
                                                 "BarcodeSequence":"CTCTCTACCAAT",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"skin",
                                                 "Description":"human skin"}},
                        {"id":"Sample6", "metadata":{
                                                 "BarcodeSequence":"CTAACTACCAAT",
                                                 "LinkerPrimerSequence":"CATGCTGCCTCCCGTAGGAGT",
                                                 "BODY_SITE":"skin",
                                                 "Description":"human skin"}}
                                ],
                     "matrix_type": "sparse",
                     "matrix_element_type": "int",
                     "shape": [5, 6],
                     "data":[[0,2,1],
                             [1,0,5],
                             [1,1,1],
                             [1,3,2],
                             [1,4,3],
                             [1,5,1],
                             [2,2,1],
                             [2,3,4],
                             [2,5,2],
                             [3,0,2],
                             [3,1,1],
                             [3,2,1],
                             [3,5,1],
                             [4,1,1],
                             [4,2,1]
                            ]
                        }"""

        self.biom = json.loads(self.biom_text)

    def test_relative_abundance(self):
        """
        Testing relative abundance() function of biom.calc.py.

        :return: Returns OK, if testing goal is achieved, otherwise raises
                error.
        """
        sample = 'Sample3'
        self.result = bc.relative_abundance(self.biom, sample)

        # List containing manual calculations
        hand_calc = [1/4.0, 1/4.0, 1/4.0, 1/4.0]

        # Obtaining list of function calculated relative abundance for sample
        result1 = self.result.values()      # result1 is a list
        result2 = result1[0]                # result2 is a dict
        # list containing the calculated relative abundance values
        func_calc = result2.values()

        # Testing the validity of relative_abundance() function.
        for hand, res in zip(hand_calc, func_calc):
            self.assertAlmostEqual(hand, res, msg='Relative abundances not calculated accurately.')

    def test_mean_otu_pct_abundance(self):
        """
        Testing mean_otu_pct_abundance() function of biom_calc.py.

        :return: Returns OK, if testing goal was achieved, otherwise raises
                error.
        """
        self.rel_a = bc.relative_abundance(self.biom, ['Sample4'])
        self.result = bc.mean_otu_pct_abundance(
            self.rel_a, ['GG_OTU_2', 'GG_OTU_3']
            )

        # Obtaining lists of function calculations and manual hand calculations
        func_calc = self.result.values()
        result1 = self.rel_a.values()       # result1 is a list
        result2 = result1[0]                # result2 is a dict
        # list containing hand calculated relative abundance values
        hand_calc = result2.values()

        # Testing the validity of the calculations of mean_otu_pct_abundance().
        for hand, res in zip(hand_calc, func_calc):
            self.assertAlmostEqual(hand*100, res, msg='Mean OTU not calculated accurately.')

    def test_MRA(self):
        """
        Testing mean relative abundance calculation, MRA() function
        of biom_calc.py.

        :return: Returns OK, if testing goal was achieved, otherwise
            raises error.
        """
        self.result = bc.MRA(self.biom, 'Sample4')
        self.mean_otu = bc.mean_otu_pct_abundance(
            bc.relative_abundance(self.biom, ['Sample4']),
            ['GG_OTU_1', 'GG_OTU_2', 'GG_OTU_3', 'GG_OUTU4', 'GG_OTU_5']
            )

        # Obtaining lists of function calculations and manual hand calculations
        func_calc = self.result.values()
        hand_calc = self.mean_otu.values()

        # Testing the validity of the calculations of mean_otu_pct_abundance().
        for hand, res in zip(hand_calc, func_calc):
            self.assertAlmostEqual(hand, res, msg='Mean OTU not calculated accurately.')

    def test_raw_abundance(self):
        """
        Testing raw_abundance() function of biom_calc.py.

        :return: Returns OK, if testing goal is achieved, otherwise raises
            error.
        """
        self.result = {}
        self.result = bc.raw_abundance(self.biom, ['Sample1', 'Sample3'])

        # Lists containing hand and function calculated values.
        hand_calc = [
            math.log(1, 10), math.log(5, 10), math.log(1, 10),
            math.log(3, 10), math.log(1, 10)
            ]
        func_calc = self.result.values()

        # Testing validity of raw_abundance() function.
        for hand, res in zip(hand_calc, func_calc):
            self.assertAlmostEqual(hand, res, msg='Raw abundances not calculated accurately!')

    def tearDown(self):
        """
        No particular event to clean, delete or close in testing biom_calc.py.
        """
        pass

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import logging
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger("geolife")
#stdout = logging.StreamHandler()
#stdout.setLevel(logging.INFO)
#logger.addHandler(stdout)

from sqlalchemy import Index
from datetime import timedelta
from datetime import time
from utils import timerange
from utils import timeAdd
from utils import ETACalculator
from utils import timeDifferenceSeconds
from utils import num_elements_in_time_range
from raw.record import GeoLifeUser
from raw.record import WRecord
from schema import HomogenizedRecord
from schema import HomogenizedGeoLifeUser
from schema import initialize_table
import argparse

import config
engine = config.getEngine()

from sqlalchemy.orm import sessionmaker
Session = sessionmaker()
Session.configure(bind=engine)

def get_users_present_on(weekday):
  session = Session()
  query = session.query(GeoLifeUser.id)
  if weekday is not None:
    result_set = query.filter(
      GeoLifeUser.weekday == weekday
    ).all()

  else:
    result_set = query.all()

  users = set()
  for u, in result_set:
    users.add(u)

  session.close()
  return users

class RecentUserRecord:
  def __init__(self, users):
    self.most_recent_record = {}

    s = Session()

    logger.info("Preloading RecentUserRecord object")
    records = s.query(WRecord).order_by(WRecord.time)
    eta = ETACalculator(len(users), name="Earliest User Records")

    for u in users:
      r = records.filter(WRecord.user == u).first()
      self.most_recent_record[u] = r
      logger.info("First record for user {0}: {1}".format(u, r))
      eta.checkpoint()
      logger.info(eta.eta())
    s.close()

  def get(self, time, user):
    logger.debug("Fetching closest record preceeding {0} for user {1}".format(
      time, user
    ))
    if not user in self.most_recent_record:
      logger.debug("Most recent record not available. Querying datebase.")
      s = Session()
      r = s.query(WRecord).filter(
        WRecord.time < t,
        WRecord.user == u
      ).order_by(WRecord.time).first()
      self.most_recent_record[user] = r
      s.close()

    else:
      r = self.most_recent_record[user]

    logger.debug("Record: {0}".format(r))
    return r

  def update(self, user, record):
    logger.debug("Updating most recent record for user {0} from {1}"
                 " to {2}".format(
      user, self.most_recent_record[user], record,
    ))
    self.most_recent_record[user] = record

def get_arguments():
  parser = argparse.ArgumentParser(
    description='Homogenize time deltas between records.'
  )
  parser.add_argument(
    '-w', '--weekday',
    dest="weekday",
    help=('Numerical indicator of weekday (0 is Monday, 1 is Tuesday, ..., 6'
          ' is Sunday). Default: None (all weekdays will be accounted for).'),
    type=int,
    default=None,
  )
  parser.add_argument(
    '-d', '--time-delta',
    dest='time_delta',
    help="Number of seconds that should be between any two consecutive records",
    type=lambda x: timedelta(seconds=int(x)),
    default=timedelta(seconds=5),
  )
  parser.add_argument(
    '-t', '--dry-run',
    dest="dry_run",
    action="store_true",
    help=('Boolean indicator controlling if data should be added to the'
          ' database (default: False).'),
  )

  args = parser.parse_args()
  return args

def verify_time_homogeniety(users, time_delta, db_session):
  logger.info("Verifying time homogeneity between {0} users".format(
    len(users)
  ))
  eta = ETACalculator(len(users), "Time Homogeneity Verification")
  for u in users:
    result_set = db_session.query(HomogenizedRecord.time)\
                           .filter(HomogenizedRecord.user == u)\
                           .order_by(HomogenizedRecord.time)
    previous = None
    i = 0
    for r, in result_set:
      if previous is not None:
        diff = timeDifferenceSeconds(r, previous)
        assert diff == time_delta.total_seconds(), (
          "Time homogeniety was not preserved for user {user}, record #{record}.\n"
          "Expected time delta: {exp}\n"
          "Actual time delta:   {act}".format(
            user=u, record=i, exp=time_delta, act=diff
        ))

      previous = r
      i += 1
    eta.checkpoint()
    logger.info(eta.eta())

if __name__ == "__main__":
  args = get_arguments()
  dry_run = args.dry_run
  weekday = args.weekday
  delta = args.time_delta

  # Create the Time Homogenized tables.
  if not dry_run:
    initialize_table(engine)
  session = Session()

  users = get_users_present_on(weekday)

  logger.debug("#"*80)
  logger.debug("Users selected: {0}".format(users))

  most_recent_records = RecentUserRecord(users)
  users_present = set()
  homogenized_records = []

  n = num_elements_in_time_range(start=time.min, end=time.max, step=delta)
  eta_til_completed_day = ETACalculator(n, "Synthesis")

  for t in timerange(time.min, time.max, delta):
      logger.debug("="*60)
      logger.debug("Querying for time {0}".format(t))
      record_set = session.query(WRecord).filter(
        WRecord.time >= t,
        WRecord.time < timeAdd(t, delta),
        WRecord.user.in_(users)
      ).order_by(WRecord.time).all()

      i = 0
      for r in record_set:
        if r.user not in users_present:
          logger.debug("-"*40)
          logger.debug("Record from DB: {0}".format(r))
          users_present.add(r.user)
          r.time = t
          homogenized_records.append(r)
          most_recent_records.update(r.user, r)
          i += 1
      logger.debug("{0} records on {1}".format(i, t))

      users_not_present = users - users_present
      logger.debug("Users not present: {0}".format(users_not_present))
      for u in users_not_present:
        logger.debug("+"*40)
        most_recent_record = most_recent_records.get(t, u)
        most_recent_record.time = t
        homogenized_records.append(most_recent_record)

      logger.info("Adding {0} homogenized records to database".format(
        len(homogenized_records)
      ))
      if not dry_run:
        session.add_all([HomogenizedRecord(
            user=r.user, latitude=r.latitude, longitude=r.longitude,
            time=r.time, weekday=r.weekday,
          ) for r in homogenized_records
        ])
        session.commit()

      users_present.clear()

      assert len(homogenized_records) == len(users), "Number of homogenized records {0} is not equal to the number of users {1}".format(len(homogenized_records), len(users))
      del homogenized_records[:]

      eta_til_completed_day.checkpoint()
      logger.info(eta_til_completed_day.eta())

  # Create the indices needed for fast performance!
  logger.info("Creating index on raw record time columns")
  if not dry_run:
    Index('homo_time', HomogenizedRecord.__table__.c.time).create(engine)
    Index('homo_user', HomogenizedRecord.__table__.c.user).create(engine)
    Index(
      'homo_usertime',
      HomogenizedRecord.__table__.c.time,
      HomogenizedRecord.__table__.c.user
    ).create(engine)

  #verify_time_homogeniety(users=users, time_delta=delta, db_session=session)

<commit_msg>tapping into the created view<commit_after>import logging
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger("geolife")
#stdout = logging.StreamHandler()
#stdout.setLevel(logging.INFO)
#logger.addHandler(stdout)

from sqlalchemy import Index
from sqlalchemy import Table
from sqlalchemy import MetaData

from datetime import timedelta
from datetime import time
from utils import timerange
from utils import timeAdd
from utils import ETACalculator
from utils import timeDifferenceSeconds
from utils import num_elements_in_time_range
from schema import HomogenizedRecord
from schema import HomogenizedGeoLifeUser
from schema import initialize_table
import argparse

import config
engine = config.getEngine()

from sqlalchemy.orm import sessionmaker
Session = sessionmaker()
Session.configure(bind=engine)

metadata = MetaData()
RecordsOnOneDay = Table(
  'day_records_view', metadata, autoload=True, autoload_with=engine
)

print("#"*40)
for c in RecordsOnOneDay.columns:
  print(c)
print("#"*40)
"""
def get_users():
  session = Session()
  query = session.query(GeoLifeUser.id)
  result_set = query.all()

  users = set()
  for u, in result_set:
    users.add(u)

  session.close()
  return users

class RecentUserRecord:
  def __init__(self, users):
    self.most_recent_record = {}

    s = Session()

    logger.info("Preloading RecentUserRecord object")
    records = s.query(WRecord).order_by(WRecord.time)
    eta = ETACalculator(len(users), name="Earliest User Records")

    for u in users:
      r = records.filter(WRecord.user == u).first()
      self.most_recent_record[u] = r
      logger.info("First record for user {0}: {1}".format(u, r))
      eta.checkpoint()
      logger.info(eta.eta())
    s.close()

  def get(self, time, user):
    logger.debug("Fetching closest record preceeding {0} for user {1}".format(
      time, user
    ))
    if not user in self.most_recent_record:
      logger.debug("Most recent record not available. Querying datebase.")
      s = Session()
      r = s.query(WRecord).filter(
        WRecord.time < t,
        WRecord.user == u
      ).order_by(WRecord.time).first()
      self.most_recent_record[user] = r
      s.close()

    else:
      r = self.most_recent_record[user]

    logger.debug("Record: {0}".format(r))
    return r

  def update(self, user, record):
    logger.debug("Updating most recent record for user {0} from {1}"
                 " to {2}".format(
      user, self.most_recent_record[user], record,
    ))
    self.most_recent_record[user] = record

def get_arguments():
  parser = argparse.ArgumentParser(
    description='Homogenize time deltas between records.'
  )
  parser.add_argument(
    '-d', '--time-delta',
    dest='time_delta',
    help="Number of seconds that should be between any two consecutive records",
    type=lambda x: timedelta(seconds=int(x)),
    default=timedelta(seconds=5),
  )
  parser.add_argument(
    '-t', '--dry-run',
    dest="dry_run",
    action="store_true",
    help=('Boolean indicator controlling if data should be added to the'
          ' database (default: False).'),
  )

  args = parser.parse_args()
  return args

def verify_time_homogeniety(users, time_delta, db_session):
  logger.info("Verifying time homogeneity between {0} users".format(
    len(users)
  ))
  eta = ETACalculator(len(users), "Time Homogeneity Verification")
  for u in users:
    result_set = db_session.query(HomogenizedRecord.time)\
                           .filter(HomogenizedRecord.user == u)\
                           .order_by(HomogenizedRecord.time)
    previous = None
    i = 0
    for r, in result_set:
      if previous is not None:
        diff = timeDifferenceSeconds(r, previous)
        assert diff == time_delta.total_seconds(), (
          "Time homogeniety was not preserved for user {user}, record #{record}.\n"
          "Expected time delta: {exp}\n"
          "Actual time delta:   {act}".format(
            user=u, record=i, exp=time_delta, act=diff
        ))

      previous = r
      i += 1
    eta.checkpoint()
    logger.info(eta.eta())

if __name__ == "__main__":
  args = get_arguments()
  dry_run = args.dry_run
  delta = args.time_delta

  # Create the Time Homogenized tables.
  if not dry_run:
    initialize_table(engine)
  session = Session()

  users = get_users()

  logger.debug("#"*80)
  logger.debug("Users selected: {0}".format(users))

  most_recent_records = RecentUserRecord(users)
  users_present = set()
  homogenized_records = []

  n = num_elements_in_time_range(start=time.min, end=time.max, step=delta)
  eta_til_completed_day = ETACalculator(n, "Synthesis")

  for t in timerange(time.min, time.max, delta):
      logger.debug("="*60)
      logger.debug("Querying for time {0}".format(t))
      record_set = session.query(WRecord).filter(
        WRecord.time >= t,
        WRecord.time < timeAdd(t, delta),
        WRecord.user.in_(users)
      ).order_by(WRecord.time).all()

      i = 0
      for r in record_set:
        if r.user not in users_present:
          logger.debug("-"*40)
          logger.debug("Record from DB: {0}".format(r))
          users_present.add(r.user)
          r.time = t
          homogenized_records.append(r)
          most_recent_records.update(r.user, r)
          i += 1
      logger.debug("{0} records on {1}".format(i, t))

      users_not_present = users - users_present
      logger.debug("Users not present: {0}".format(users_not_present))
      for u in users_not_present:
        logger.debug("+"*40)
        most_recent_record = most_recent_records.get(t, u)
        most_recent_record.time = t
        homogenized_records.append(most_recent_record)

      logger.info("Adding {0} homogenized records to database".format(
        len(homogenized_records)
      ))
      if not dry_run:
        session.add_all([HomogenizedRecord(
            user=r.user, latitude=r.latitude, longitude=r.longitude,
            time=r.time, weekday=r.weekday,
          ) for r in homogenized_records
        ])
        session.commit()

      users_present.clear()

      assert len(homogenized_records) == len(users), "Number of homogenized records {0} is not equal to the number of users {1}".format(len(homogenized_records), len(users))
      del homogenized_records[:]

      eta_til_completed_day.checkpoint()
      logger.info(eta_til_completed_day.eta())

  # Create the indices needed for fast performance!
  logger.info("Creating index on raw record time columns")
  if not dry_run:
    Index('homo_time', HomogenizedRecord.__table__.c.time).create(engine)
    Index('homo_user', HomogenizedRecord.__table__.c.user).create(engine)
    Index(
      'homo_usertime',
      HomogenizedRecord.__table__.c.time,
      HomogenizedRecord.__table__.c.user
    ).create(engine)

  #verify_time_homogeniety(users=users, time_delta=delta, db_session=session)
"""
<|endoftext|>


<commit_before>import logging
import os

from django.utils import simplejson

from google.appengine.api import memcache
from google.appengine.ext import db, webapp
from google.appengine.ext.webapp import template, util

from helpers.api_helper import ApiHelper
from models import Event, EventTeam, Match, Team

class ApiTeamsShow(webapp.RequestHandler):
    """
    Information about teams.
    """
    def get(self):
        teams = list()
        team_keys = self.request.get('teams').split(',')

        for team_key in team_keys:
            teams.append(ApiHelper.getTeamInfo(team_key))

        self.response.out.write(simplejson.dumps(teams))

class ApiTeamDetails(webapp.RequestHandler):
    """
    Information about a Team in a particular year, including full Event and Match objects
    """
    def get(self):
        team_key = self.request.get('team')
        year = self.request.get('year')

        team_dict = ApiHelper.getTeamInfo(team_key)
        if self.request.get('events'):
            team_dict = ApiHelper.addTeamEvents(team_dict, year)

        #TODO: matches

        self.response.out.write(simplejson.dumps(team_dict))

class ApiEventsShow(webapp.RequestHandler):
    """
    Information about events.
    Deprecation notice. Please use ApiEventList, or ApiEventDetails.
    """
    def get(self):
        logging.warning("Deprecation notice: ApiEventsShow.")

        event_keys = set()

        if self.request.get("year"):
            events = Event.all().filter("year =", int(self.request.get("year"))).fetch(500)
            event_keys = event_keys.union(set([event.key().name() for event in events]))

        event_keys = filter(None, event_keys.union(set(self.request.get("events").split(','))))
        events = [ApiHelper.getEventInfo(event_key) for event_key in event_keys]

        self.response.out.write(simplejson.dumps(events))

class ApiEventList(webapp.RequestHandler):
    """
    List of teams for a year with key_name, team number, and name.
    """

    def get(self):
        try:
            year = int(self.request.get("year"))
        except ValueError:
            error_message = {"Parameter Error": "'year' is a required parameter."}
            self.response.out.write(simplejson.dumps(error_message))
            return False

        memcache_key = "api_event_list_%s" % year
        event_list = memcache.get(memcache_key)

        if event_list is None:
            event_list = []
            events = Event.all().filter("year =", year).fetch(500)
            for event in events:
                event_dict = dict()
                event_dict["key"] = event.key().name()
                event_dict["name"] = event.name
                event_dict["event_code"] = event.short_name
                event_dict["official"] = event.official
                event_list.append(event_dict)
            memcache.set(memcache_key, event_list, 600)

        self.response.out.write(simplejson.dumps(event_list))

class ApiEventDetails(webapp.RequestHandler):
    """
    Return a specifc event with details.
    """

    def get(self):
        event_key = str(self.request.get("event"))
        if event_key is "" or event_key is None:
            error_message = {"Parameter Error": "'event' is a required parameter."}
            self.response.out.write(simplejson.dumps(error_message))
            return False

        event_dict = ApiHelper.getEventInfo(event_key)

        self.response.out.write(simplejson.dumps(event_dict))
<commit_msg>Added start and end date to event list.<commit_after>import logging
import os

from django.utils import simplejson

from google.appengine.api import memcache
from google.appengine.ext import db, webapp
from google.appengine.ext.webapp import template, util

from helpers.api_helper import ApiHelper
from models import Event, EventTeam, Match, Team

class ApiTeamsShow(webapp.RequestHandler):
    """
    Information about teams.
    """
    def get(self):
        teams = list()
        team_keys = self.request.get('teams').split(',')

        for team_key in team_keys:
            teams.append(ApiHelper.getTeamInfo(team_key))

        self.response.out.write(simplejson.dumps(teams))

class ApiTeamDetails(webapp.RequestHandler):
    """
    Information about a Team in a particular year, including full Event and Match objects
    """
    def get(self):
        team_key = self.request.get('team')
        year = self.request.get('year')

        team_dict = ApiHelper.getTeamInfo(team_key)
        if self.request.get('events'):
            team_dict = ApiHelper.addTeamEvents(team_dict, year)

        #TODO: matches

        self.response.out.write(simplejson.dumps(team_dict))

class ApiEventsShow(webapp.RequestHandler):
    """
    Information about events.
    Deprecation notice. Please use ApiEventList, or ApiEventDetails.
    """
    def get(self):
        logging.warning("Deprecation notice: ApiEventsShow.")

        event_keys = set()

        if self.request.get("year"):
            events = Event.all().filter("year =", int(self.request.get("year"))).fetch(500)
            event_keys = event_keys.union(set([event.key().name() for event in events]))

        event_keys = filter(None, event_keys.union(set(self.request.get("events").split(','))))
        events = [ApiHelper.getEventInfo(event_key) for event_key in event_keys]

        self.response.out.write(simplejson.dumps(events))

class ApiEventList(webapp.RequestHandler):
    """
    List of teams for a year with key_name, team number, and name.
    """

    def get(self):
        try:
            year = int(self.request.get("year"))
        except ValueError:
            error_message = {"Parameter Error": "'year' is a required parameter."}
            self.response.out.write(simplejson.dumps(error_message))
            return False

        memcache_key = "api_event_list_%s" % year
        event_list = memcache.get(memcache_key)

        if event_list is None:
            event_list = []
            events = Event.all().filter("year =", year).fetch(500)
            for event in events:
                event_dict = dict()
                event_dict["key"] = event.key().name()
                event_dict["name"] = event.name
                event_dict["event_code"] = event.short_name
                event_dict["official"] = event.official

                if event.start_date:
                    event_dict["start_date"] = event.start_date.isoformat()
                else:
                    event_dict["start_date"] = None
                if event.end_date:
                    event_dict["end_date"] = event.end_date.isoformat()
                else:
                    event_dict["end_date"] = None

                event_list.append(event_dict)
            memcache.set(memcache_key, event_list, 600)

        self.response.out.write(simplejson.dumps(event_list))

class ApiEventDetails(webapp.RequestHandler):
    """
    Return a specifc event with details.
    """

    def get(self):
        event_key = str(self.request.get("event"))
        if event_key is "" or event_key is None:
            error_message = {"Parameter Error": "'event' is a required parameter."}
            self.response.out.write(simplejson.dumps(error_message))
            return False

        event_dict = ApiHelper.getEventInfo(event_key)

        self.response.out.write(simplejson.dumps(event_dict))
<|endoftext|>