<commit_before># -*- coding: utf-8 -*-

"""
Make API calls.

.. moduleauthor:: Jiwen Xin <kevinxin@scripps.edu>

"""
from collections import defaultdict
from copy import deepcopy
from itertools import groupby
from operator import itemgetter
from .registry import Registry
from .apicall import BioThingsCaller
from .api_output_parser import OutputParser
from .config import metadata
from .api_preprocess import APIPreprocess

BIOTHINGS_APIs = [k for k, v in metadata.items() if v.get("api_type") == 'biothings']

class Dispatcher():

    """Dispatch API calls."""

    def __init__(self, registry=None):
        """Load BTE registry and API caller."""
        if not registry:
            self.registry = Registry().registry
        else:
            self.registry = registry.registry
        self.caller = BioThingsCaller()

    @staticmethod
    def get_unique_edge_id(edge):
        operation = edge['operation']
        _id = '-'.join([str(edge['value']), operation['server'], operation['method']])
        request_body = operation.get("requestBody")
        if request_body and request_body.get("body"):
            for k in sorted(request_body.get('body').keys()):
                _id += ('-' + k + '-' + str(request_body.get("body")[k]))
        parameters = operation.get("parameters")
        if parameters:
            for k in sorted(parameters.keys()):
                _id += ('-' + k + '-' + str(parameters[k]))
        return _id

    @staticmethod
    def get_all_edges(query_id2inputs_mapping):
        res = []
        for v in query_id2inputs_mapping.values():
            if isinstance(v, list) and len(v) > 0:
                res.append(v[0])
        return res

    @staticmethod
    def group_edges(edges):
        grouper = itemgetter("operation_id")
        new_edges = []
        # group all edges based on their API and input_field value
        for _, grp in groupby(sorted(edges, key=grouper), grouper):
            grp = list(grp)
            new_edge = deepcopy(grp[0])
            values = set()
            for edge in grp:
                if isinstance(edge['value'], list):
                    values |= set(edge['value'])
                else:
                    values.add(edge['value'])
            new_edge['value'] = values
            new_edges.append(new_edge)
        return new_edges

    @staticmethod
    def subset_mapping_file(edges, mapping_file):
        """Only maintain a subset of mapping file based on edge label."""
        mapping_keys = [_item.get('mapping_key') for _item in edges]
        if mapping_keys:
            mapping_keys += ["@type", "@context"]
            return {k: v for (k, v) in mapping_file.items() if k in mapping_keys}
        return mapping_file

    def construct_api_calls(self, edges):
        """Construct API calls for apicall module using edge groups."""
        unique_edge_ids = set()
        edge_id2query_id_mapping = {}
        query_id2inputs_mapping = defaultdict(list)
        edges = self.group_edges(edges)
        for edge in edges:
            api = edge['api']
            if edge['operation'].get('supportBatch'):
                edge['value'] = edge['operation']['inputSeparator'].join(edge['value'])
                edge_id = self.get_unique_edge_id(edge)
                if edge_id in unique_edge_ids:
                    edge['internal_query_id'] = edge_id2query_id_mapping[edge_id]
                else:
                    internal_query_id = 'API ' + self.api_dict[api]['num'] + '.' + str(self.api_dict[api]['alphas'].pop(0))
                    edge['internal_query_id'] = internal_query_id
                    edge_id2query_id_mapping[edge_id] = internal_query_id
                    unique_edge_ids.add(edge_id)
                query_id2inputs_mapping[internal_query_id].append(edge)
                # internal_query_id += 1
            else:
                for val in edge['value']:
                    new_edge = deepcopy(edge)
                    new_edge['value'] = val
                    edge_id = self.get_unique_edge_id(new_edge)
                    if edge_id in unique_edge_ids:
                        new_edge['internal_query_id'] = edge_id2query_id_mapping[edge_id]
                    else:
                        internal_query_id = 'API ' + self.api_dict[api]['num'] + '.' + str(self.api_dict[api]['alphas'].pop(0))
                        new_edge['internal_query_id'] = internal_query_id
                        edge_id2query_id_mapping[edge_id] = internal_query_id
                        unique_edge_ids.add(edge_id)
                    query_id2inputs_mapping[internal_query_id].append(new_edge)
                    # internal_query_id += 1
        return query_id2inputs_mapping

    @staticmethod
    def add_metadata_to_output(operation, res, output_id):
        if isinstance(res, dict):
            if output_id not in res:
                return []
            if not isinstance(res[output_id], list):
                res[output_id] = [res[output_id]]
            new_res = []
            for val in res[output_id]:
                tmp = deepcopy(res)
                tmp[output_id] = val
                tmp.update(
                    {
                        "$api": operation['api_name'],
                        "$source": operation.get("source"),
                        "@type": operation['output_type']
                    }
                )
                new_res.append(tmp)
            return new_res
        res = {
            "$api": operation['api_name'],
            "$source": operation.get("source"),
            "@type": operation['output_type'],
            operation['output_id']: [res]
        }
        return [res]

    @staticmethod
    def count_hits(res):
        cnt = 0
        if not res:
            return cnt
        for pred_infos in res.values():
            if pred_infos:
                for info in pred_infos.values():
                    cnt += len(info)
        return cnt

    def dispatch(self, edges, verbose=False):
        """Send request to and parse response from API."""
        results = {}
        self.unique_apis = {_edge['api'] for _edge in edges if _edge}
        self.api_dict = {}
        for i, _api in enumerate(list(self.unique_apis)):
            self.api_dict[_api] = {'alphas': list(range(1, 10000)), 'num': str(i + 1)}
        query_id2inputs_mapping = self.construct_api_calls(edges)
        # print(apis, inputs, modes, vals, grped_edges)
        responses, self.log = self.caller.call_apis(self.get_all_edges(query_id2inputs_mapping), verbose=verbose)
        if verbose:
            print("\n\n==== Step #3: Output normalization ====\n")
        self.log.append("\n\n==== Step #3: Output normalization ====\n")
        for response in responses:
            if response['result'] == {}:
                continue
            output_types = []
            query_id = response['internal_query_id']
            total_hits = 0
            for edge in query_id2inputs_mapping[query_id]:
                operation = edge['operation']
                api_name = edge['api']
                if api_name[:4] in ['semm', 'cord']:
                    output_types = [edge['output_type']]
                _res = APIPreprocess(response['result'], operation['api_type'], api_name, output_types).restructure()
                mapping = operation['response_mapping']
                _res = OutputParser(_res, mapping, operation['supportBatch'], api_name, operation['api_type']).parse()
                if not operation['supportBatch']:
                    # preprocess biolink results
                    # if val is not present in results dict and _res is not empty
                    if not _res:
                        continue
                    val = edge['value']
                    if val not in results:
                        results[val] = {}
                    # loop through API call response
                    for k, v in _res.items():
                        # if key is not present in final res, create a list
                        if k not in results[val]:
                            results[val][k] = []
                        if isinstance(v, list):
                            for _v in v:
                                _v = self.add_metadata_to_output(operation, _v, operation['output_id'])
                                total_hits += len(_v)
                                results[val][k] += _v
                        else:
                            v = self.add_metadata_to_output(operation, v, operation['output_id'])
                            total_hits += len(v)
                            results[val][k] += v
                else:
                    if not _res:
                        continue
                    for m, n in _res.items():
                        if m not in results:
                            results[m] = {}
                        for k, v in n.items():
                            if k not in results[m]:
                                results[m][k] = []
                            if isinstance(v, list):
                                for _v in v:
                                    _v = self.add_metadata_to_output(operation, _v, operation['output_id'])
                                    total_hits += len(_v)
                                    results[m][k] += _v
                            elif isinstance(v, dict):
                                v = self.add_metadata_to_output(operation, v, operation['output_id'])
                                total_hits += len(v)
                                results[m][k] += v
                            else:
                                if k == 'query':
                                    continue
                                v = self.add_metadata_to_output(operation, v, operation['output_id'])
                                total_hits += len(v)
                                results[m][k] += v
            if verbose:
                if total_hits > 0:
                    print("{} {}: {} hits".format(query_id, api_name, total_hits))
                else:
                    print("{} {}: No hits".format(query_id, api_name))
            if total_hits > 0:
                self.log.append("{} {}: {} hits".format(query_id, api_name, total_hits))
            else:
                self.log.append("{} {}: No hits".format(query_id, api_name))
        return (dict(results), self.log)
<commit_msg>fix logic error ;<commit_after># -*- coding: utf-8 -*-

"""
Make API calls.

.. moduleauthor:: Jiwen Xin <kevinxin@scripps.edu>

"""
from collections import defaultdict
from copy import deepcopy
from itertools import groupby
from operator import itemgetter
from .registry import Registry
from .apicall import BioThingsCaller
from .api_output_parser import OutputParser
from .config import metadata
from .api_preprocess import APIPreprocess

BIOTHINGS_APIs = [k for k, v in metadata.items() if v.get("api_type") == 'biothings']

class Dispatcher():

    """Dispatch API calls."""

    def __init__(self, registry=None):
        """Load BTE registry and API caller."""
        if not registry:
            self.registry = Registry().registry
        else:
            self.registry = registry.registry
        self.caller = BioThingsCaller()

    @staticmethod
    def get_unique_edge_id(edge):
        operation = edge['operation']
        _id = '-'.join([str(edge['value']), operation['server'], operation['method']])
        request_body = operation.get("requestBody")
        if request_body and request_body.get("body"):
            for k in sorted(request_body.get('body').keys()):
                _id += ('-' + k + '-' + str(request_body.get("body")[k]))
        parameters = operation.get("parameters")
        if parameters:
            for k in sorted(parameters.keys()):
                _id += ('-' + k + '-' + str(parameters[k]))
        return _id

    @staticmethod
    def get_all_edges(query_id2inputs_mapping):
        res = []
        for v in query_id2inputs_mapping.values():
            if isinstance(v, list) and len(v) > 0:
                res.append(v[0])
        return res

    @staticmethod
    def group_edges(edges):
        grouper = itemgetter("operation_id")
        new_edges = []
        # group all edges based on their API and input_field value
        for _, grp in groupby(sorted(edges, key=grouper), grouper):
            grp = list(grp)
            new_edge = deepcopy(grp[0])
            values = set()
            for edge in grp:
                if isinstance(edge['value'], list):
                    values |= set(edge['value'])
                else:
                    values.add(edge['value'])
            new_edge['value'] = values
            new_edges.append(new_edge)
        return new_edges

    @staticmethod
    def subset_mapping_file(edges, mapping_file):
        """Only maintain a subset of mapping file based on edge label."""
        mapping_keys = [_item.get('mapping_key') for _item in edges]
        if mapping_keys:
            mapping_keys += ["@type", "@context"]
            return {k: v for (k, v) in mapping_file.items() if k in mapping_keys}
        return mapping_file

    def construct_api_calls(self, edges):
        """Construct API calls for apicall module using edge groups."""
        unique_edge_ids = set()
        edge_id2query_id_mapping = {}
        query_id2inputs_mapping = defaultdict(list)
        edges = self.group_edges(edges)
        for edge in edges:
            api = edge['api']
            if edge['operation'].get('supportBatch'):
                edge['value'] = edge['operation']['inputSeparator'].join(edge['value'])
                edge_id = self.get_unique_edge_id(edge)
                if edge_id in unique_edge_ids:
                    internal_query_id = edge_id2query_id_mapping[edge_id]
                else:
                    internal_query_id = 'API ' + self.api_dict[api]['num'] + '.' + str(self.api_dict[api]['alphas'].pop(0))
                    edge['internal_query_id'] = internal_query_id
                    edge_id2query_id_mapping[edge_id] = internal_query_id
                    unique_edge_ids.add(edge_id)
                edge['internal_query_id'] = internal_query_id
                query_id2inputs_mapping[internal_query_id].append(edge)
                # internal_query_id += 1
            else:
                for val in edge['value']:
                    new_edge = deepcopy(edge)
                    new_edge['value'] = val
                    edge_id = self.get_unique_edge_id(new_edge)
                    if edge_id in unique_edge_ids:
                        internal_query_id = edge_id2query_id_mapping[edge_id]
                    else:
                        internal_query_id = 'API ' + self.api_dict[api]['num'] + '.' + str(self.api_dict[api]['alphas'].pop(0))
                        edge_id2query_id_mapping[edge_id] = internal_query_id
                        unique_edge_ids.add(edge_id)
                    new_edge['internal_query_id'] = internal_query_id
                    query_id2inputs_mapping[internal_query_id].append(new_edge)
                    # internal_query_id += 1
        return query_id2inputs_mapping

    @staticmethod
    def add_metadata_to_output(operation, res, output_id):
        if isinstance(res, dict):
            if output_id not in res:
                return []
            if not isinstance(res[output_id], list):
                res[output_id] = [res[output_id]]
            new_res = []
            for val in res[output_id]:
                tmp = deepcopy(res)
                tmp[output_id] = val
                tmp.update(
                    {
                        "$api": operation['api_name'],
                        "$source": operation.get("source"),
                        "@type": operation['output_type']
                    }
                )
                new_res.append(tmp)
            return new_res
        res = {
            "$api": operation['api_name'],
            "$source": operation.get("source"),
            "@type": operation['output_type'],
            operation['output_id']: [res]
        }
        return [res]

    @staticmethod
    def count_hits(res):
        cnt = 0
        if not res:
            return cnt
        for pred_infos in res.values():
            if pred_infos:
                for info in pred_infos.values():
                    cnt += len(info)
        return cnt

    def dispatch(self, edges, verbose=False):
        """Send request to and parse response from API."""
        results = {}
        self.unique_apis = {_edge['api'] for _edge in edges if _edge}
        self.api_dict = {}
        for i, _api in enumerate(list(self.unique_apis)):
            self.api_dict[_api] = {'alphas': list(range(1, 10000)), 'num': str(i + 1)}
        query_id2inputs_mapping = self.construct_api_calls(edges)
        # print(apis, inputs, modes, vals, grped_edges)
        responses, self.log = self.caller.call_apis(self.get_all_edges(query_id2inputs_mapping), verbose=verbose)
        if verbose:
            print("\n\n==== Step #3: Output normalization ====\n")
        self.log.append("\n\n==== Step #3: Output normalization ====\n")
        for response in responses:
            if response['result'] == {}:
                continue
            output_types = []
            query_id = response['internal_query_id']
            total_hits = 0
            for edge in query_id2inputs_mapping[query_id]:
                operation = edge['operation']
                api_name = edge['api']
                if api_name[:4] in ['semm', 'cord']:
                    output_types = [edge['output_type']]
                _res = APIPreprocess(response['result'], operation['api_type'], api_name, output_types).restructure()
                mapping = operation['response_mapping']
                _res = OutputParser(_res, mapping, operation['supportBatch'], api_name, operation['api_type']).parse()
                if not operation['supportBatch']:
                    # preprocess biolink results
                    # if val is not present in results dict and _res is not empty
                    if not _res:
                        continue
                    val = edge['value']
                    if val not in results:
                        results[val] = {}
                    # loop through API call response
                    for k, v in _res.items():
                        # if key is not present in final res, create a list
                        if k not in results[val]:
                            results[val][k] = []
                        if isinstance(v, list):
                            for _v in v:
                                _v = self.add_metadata_to_output(operation, _v, operation['output_id'])
                                total_hits += len(_v)
                                results[val][k] += _v
                        else:
                            v = self.add_metadata_to_output(operation, v, operation['output_id'])
                            total_hits += len(v)
                            results[val][k] += v
                else:
                    if not _res:
                        continue
                    for m, n in _res.items():
                        if m not in results:
                            results[m] = {}
                        for k, v in n.items():
                            if k not in results[m]:
                                results[m][k] = []
                            if isinstance(v, list):
                                for _v in v:
                                    _v = self.add_metadata_to_output(operation, _v, operation['output_id'])
                                    total_hits += len(_v)
                                    results[m][k] += _v
                            elif isinstance(v, dict):
                                v = self.add_metadata_to_output(operation, v, operation['output_id'])
                                total_hits += len(v)
                                results[m][k] += v
                            else:
                                if k == 'query':
                                    continue
                                v = self.add_metadata_to_output(operation, v, operation['output_id'])
                                total_hits += len(v)
                                results[m][k] += v
            if verbose:
                if total_hits > 0:
                    print("{} {}: {} hits".format(query_id, api_name, total_hits))
                else:
                    print("{} {}: No hits".format(query_id, api_name))
            if total_hits > 0:
                self.log.append("{} {}: {} hits".format(query_id, api_name, total_hits))
            else:
                self.log.append("{} {}: No hits".format(query_id, api_name))
        return (dict(results), self.log)
<|endoftext|>


<commit_before>import numpy as np
import csv

def normalize(X):
    mn = np.min(X, axis=0)
    mx = np.max(X, axis=0)
    rn = mx - mn
    X = np.array([(r - mn)/rn for r in X])
    return X

def log_transform(X, norm=False):
    X = np.log(normalize(X) + 1) if norm else np.log(X + 1)
    return X

def sqrt_transform(X, norm=False):
    X = np.sqrt(normalize(X)) if norm else np.sqrt(X)
    return X

def main():
    with open('/Users/Tyler/synapsinR_7thA.tif.Pivots.txt.2011Features.txt?dl=0','r') as csvfile:
        reader = csv.reader(csvfile, delimiter=',')
        X = np.array([[float(e) for e in r] for r in reader])
        Xlog = log_transform(X)
        Xsq = sqrt_transform(X)
        np.save('/Users/Tyler/Google Drive/DataScience/synapse_features_log.npy',Xlog)
        np.save('/Users/Tyler/Google Drive/DataScience/synapse_features_sqrt.npy',Xsq)
        Xlog = log_transform(X, norm=True)
        Xsq = sqrt_transform(X, norm=True)
        np.save('/Users/Tyler/Google Drive/DataScience/synapse_features_log_normalized.npy',Xlog)
        np.save('/Users/Tyler/Google Drive/DataScience/synapse_features_sqrt_normalized.npy',Xsq)

if __name__ == "__main__":
    main()
<commit_msg>modified script for data transformations<commit_after>import numpy as np
import csv

def normalize(X):
    mn = np.min(X, axis=0)
    mx = np.max(X, axis=0)
    rn = mx - mn
    X = np.array([(r - mn)/rn for r in X])
    return X

def log_transform(X, norm=False):
    X = np.log(normalize(X) + 1) if norm else np.log(X + 1)
    return X

def sqrt_transform(X, norm=False):
    X = np.sqrt(normalize(X)) if norm else np.sqrt(X)
    return X

def main():
    with open('/Users/Tyler/synapsinR_7thA.tif.Pivots.txt.2011Features.txt?dl=0','r') as csvfile:
        reader = csv.reader(csvfile, delimiter=',')
        X = np.array([[float(e) for e in r] for r in reader])
        Xlog = log_transform(X)
        Xsq = sqrt_transform(X)
        np.save('/Users/Tyler/synapse_data/synapse_features_log.npy',Xlog)
        np.save('/Users/Tyler/synapse_data/synapse_features_sqrt.npy',Xsq)
        Xlog = log_transform(X, norm=True)
        Xsq = sqrt_transform(X, norm=True)
        np.save('/Users/Tyler/synapse_data/synapse_features_log_normalized.npy',Xlog)
        np.save('/Users/Tyler/synapse_data/synapse_features_sqrt_normalized.npy',Xsq)

def main2():
    with open('/Users/Tyler/synapsinR_7thA.tif.Pivots.txt.2011Features.txt?dl=0','r') as csvfile:
        reader = csv.reader(csvfile, delimiter=',')
        X = np.array([[float(e) for e in r] for r in reader])
        X = X[:,0:24]
        np.save('/Users/Tyler/synapse_data/synapse_f0_features.npy',X)

if __name__ == "__main__":
#    main()
    main2()
<|endoftext|>


<commit_before>from director.shallowCopy import shallowCopy
import director.vtkAll as vtk
from vtk.util import numpy_support
import numpy as np

def numpyToPolyData(pts, pointData=None, createVertexCells=True):

    pd = vtk.vtkPolyData()
    pd.SetPoints(getVtkPointsFromNumpy(pts.copy()))

    if pointData is not None:
        for key, value in pointData.iteritems():
            addNumpyToVtk(pd, value.copy(), key)

    if createVertexCells:
        f = vtk.vtkVertexGlyphFilter()
        f.SetInputData(pd)
        f.Update()
        pd = shallowCopy(f.GetOutput())

    return pd

def getNumpyFromVtk(dataObj, arrayName='Points', arrayType='points'):
    assert arrayType in ('points', 'cells')

    if arrayName == 'Points':
        vtkArray = dataObj.GetPoints().GetData()
    elif arrayType == 'points':
        vtkArray = dataObj.GetPointData().GetArray(arrayName)
    else:
        vtkArray = dataObj.GetCellData().GetArray(arrayName)

    if not vtkArray:
        raise KeyError('Array not found')

    return numpy_support.vtk_to_numpy(vtkArray)

def getVtkPointsFromNumpy(numpyArray):

    points = vtk.vtkPoints()
    points.SetData(getVtkFromNumpy(numpyArray))
    return points

def getVtkPolyDataFromNumpyPoints(points):
    '''
    Given an Nx3 array of xyz points
    Return a new vtkPolyData containing points and vertex cells.
    If the input points is not float64 it will be converted first.
    '''

    if points.dtype != np.float64:
        points = points.astype(np.float64)

    polyData = vtk.vtkPolyData()
    polyData.SetPoints(getVtkPointsFromNumpy(points))
    vtk.vtkPCLConversions.AddVertexCells(polyData)
    return polyData

def getVtkFromNumpy(numpyArray):

    def MakeCallback(numpyArray):
        def Closure(caller, event):
            closureArray = numpyArray
        return Closure

    vtkArray = numpy_support.numpy_to_vtk(numpyArray)
    vtkArray.AddObserver('DeleteEvent', MakeCallback(numpyArray))
    return vtkArray

def addNumpyToVtk(dataObj, numpyArray, arrayName, arrayType='points'):
    assert arrayType in ('points', 'cells')
    vtkArray = getVtkFromNumpy(numpyArray)
    vtkArray.SetName(arrayName)
    if arrayType == 'points':
        assert dataObj.GetNumberOfPoints() == numpyArray.shape[0]
        dataObj.GetPointData().AddArray(vtkArray)
    else:
        assert dataObj.GetNumberOfCells() == numpyArray.shape[0]
        dataObj.GetCellData().AddArray(vtkArray)
<commit_msg>replace deprecated implementation with numpyToPolyData<commit_after>from director.shallowCopy import shallowCopy
import director.vtkAll as vtk
from vtk.util import numpy_support
import numpy as np

def numpyToPolyData(pts, pointData=None, createVertexCells=True):

    pd = vtk.vtkPolyData()
    pd.SetPoints(getVtkPointsFromNumpy(pts.copy()))

    if pointData is not None:
        for key, value in pointData.iteritems():
            addNumpyToVtk(pd, value.copy(), key)

    if createVertexCells:
        f = vtk.vtkVertexGlyphFilter()
        f.SetInputData(pd)
        f.Update()
        pd = shallowCopy(f.GetOutput())

    return pd

def getNumpyFromVtk(dataObj, arrayName='Points', arrayType='points'):
    assert arrayType in ('points', 'cells')

    if arrayName == 'Points':
        vtkArray = dataObj.GetPoints().GetData()
    elif arrayType == 'points':
        vtkArray = dataObj.GetPointData().GetArray(arrayName)
    else:
        vtkArray = dataObj.GetCellData().GetArray(arrayName)

    if not vtkArray:
        raise KeyError('Array not found')

    return numpy_support.vtk_to_numpy(vtkArray)

def getVtkPointsFromNumpy(numpyArray):

    points = vtk.vtkPoints()
    points.SetData(getVtkFromNumpy(numpyArray))
    return points

def getVtkPolyDataFromNumpyPoints(points):
    return numpyToPolyData(points)

def getVtkFromNumpy(numpyArray):

    def MakeCallback(numpyArray):
        def Closure(caller, event):
            closureArray = numpyArray
        return Closure

    vtkArray = numpy_support.numpy_to_vtk(numpyArray)
    vtkArray.AddObserver('DeleteEvent', MakeCallback(numpyArray))
    return vtkArray

def addNumpyToVtk(dataObj, numpyArray, arrayName, arrayType='points'):
    assert arrayType in ('points', 'cells')
    vtkArray = getVtkFromNumpy(numpyArray)
    vtkArray.SetName(arrayName)
    if arrayType == 'points':
        assert dataObj.GetNumberOfPoints() == numpyArray.shape[0]
        dataObj.GetPointData().AddArray(vtkArray)
    else:
        assert dataObj.GetNumberOfCells() == numpyArray.shape[0]
        dataObj.GetCellData().AddArray(vtkArray)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Get Tox21 challenge datasets.

Notes:
* Some SMILES strings are represented by multiple compounds, with some overlap
of assays. These compounds need to be condensed and their assay outcomes need
to be reconciled.
* For compounds with activities that do not agree...look at the assays if
possible.
"""

__author__ = "Steven Kearnes"
__copyright__ = "Copyright 2014, Stanford University"
__license__ = "3-clause BSD"

import argparse
import cPickle
import gzip
import numpy as np

from rdkit import Chem

from rdkit_utils import serial

def get_args():
    """
    Get command-line arguments.
    """
    parser = argparse.ArgumentParser()
    parser.add_argument('input',
                        help='Input filename.')
    parser.add_argument('--merge', choices=['max', 'min', 'majority-',
                                            'majority+'],
                        help='Target merge strategy.')
    return parser.parse_args()

def main(input_filename, merge):
    """
    Get Tox21 chellenge datasets.

    Parameters
    ----------
    input_filename : str
        Input filename.
    merge : str
        Target merge strategy.
    """
    reader = serial.MolReader()
    reader.open(input_filename)
    dataset_names = ['NR-AR', 'NR-AhR', 'NR-AR-LBD', 'NR-ER', 'NR-ER-LBD',
                     'NR-Aromatase', 'NR-PPAR-gamma', 'SR-ARE', 'SR-ATAD5',
                     'SR-HSE', 'SR-MMP', 'SR-p53']
    data = {dataset: {} for dataset in dataset_names}
    skipped = []
    for mol in reader.get_mols():
        smiles = Chem.MolToSmiles(Chem.RemoveHs(mol), isomericSmiles=True,
                                  canonical=True)
        for prop in list(mol.GetPropNames()):
            if prop in dataset_names:
                score = int(mol.GetProp(prop))
                if smiles not in data[prop]:
                    data[prop][smiles] = []
                data[prop][smiles].append(score)
            else:  # make sure we don't miss anything important
                if prop not in skipped:
                    skipped.append(prop)
                    print "Skipping property '{}'".format(prop)
                continue

    # reconcile activity differences
    for dataset in dataset_names:
        for smiles, targets in data[dataset].items():
            targets = np.asarray(targets, dtype=int)
            if len(targets) == 1:
                data[dataset][smiles] = targets[0]
            elif np.all(targets == 0):
                data[dataset][smiles] = 0
            elif np.all(targets == 1):
                data[dataset][smiles] = 1
            else:
                if merge == 'max':
                    data[dataset][smiles] = max(targets)
                elif merge == 'min':
                    data[dataset][smiles] = min(targets)
                elif merge == 'majority-':  # 0.5 rounds down
                    data[dataset][smiles] = int(np.round(np.mean(targets)))
                elif merge == 'majority+':  # 0.5 rounds up
                    data[dataset][smiles] = (int(np.round(
                        np.mean(targets) + 1)) - 1)

    # save individual datasets
    for dataset in dataset_names:
        assert len(data[dataset])  # check for at least one molecule

        # build mol and target arrays
        mols = []
        targets = []
        for mol, target in data[dataset].items():
            mols.append(mol)
            targets.append(target)
        targets = np.asarray(targets, dtype=int)
        pos = np.count_nonzero(targets == 1)
        neg = np.count_nonzero(targets == 0)
        assert pos + neg == targets.size
        print '{}\t{}\t{}'.format(dataset, pos, neg)
        with gzip.open('{}.ism.gz'.format(dataset), 'wb') as f:
            for i, mol in enumerate(mols):
                f.write('{}\t{}\n'.format(mol, '{}-{}'.format(dataset, i)))
        with open('{}-targets.pkl'.format(dataset), 'wb') as f:
            cPickle.dump(targets, f, cPickle.HIGHEST_PROTOCOL)

if __name__ == '__main__':
    args = get_args()
    main(args.input, args.merge)
<commit_msg>Add merge options<commit_after>#!/usr/bin/env python
"""
Get Tox21 challenge datasets.

Notes:
* Some SMILES strings are represented by multiple compounds, with some overlap
of assays. These compounds need to be condensed and their assay outcomes need
to be reconciled.
* For compounds with activities that do not agree...look at the assays if
possible.
"""

__author__ = "Steven Kearnes"
__copyright__ = "Copyright 2014, Stanford University"
__license__ = "3-clause BSD"

import argparse
import cPickle
import gzip
import numpy as np

from rdkit import Chem

from rdkit_utils import serial

def get_args():
    """
    Get command-line arguments.
    """
    parser = argparse.ArgumentParser()
    parser.add_argument('input',
                        help='Input filename.')
    parser.add_argument('--merge', choices=['max', 'min', 'majority-',
                                            'majority+'], required=1,
                        help='Target merge strategy.')
    return parser.parse_args()

def main(input_filename, merge):
    """
    Get Tox21 chellenge datasets.

    Parameters
    ----------
    input_filename : str
        Input filename.
    merge : str
        Target merge strategy.
    """
    reader = serial.MolReader()
    reader.open(input_filename)
    dataset_names = ['NR-AR', 'NR-AhR', 'NR-AR-LBD', 'NR-ER', 'NR-ER-LBD',
                     'NR-Aromatase', 'NR-PPAR-gamma', 'SR-ARE', 'SR-ATAD5',
                     'SR-HSE', 'SR-MMP', 'SR-p53']
    data = {dataset: {} for dataset in dataset_names}
    skipped = []
    for mol in reader.get_mols():
        smiles = Chem.MolToSmiles(Chem.RemoveHs(mol), isomericSmiles=True,
                                  canonical=True)
        for prop in list(mol.GetPropNames()):
            if prop in dataset_names:
                score = int(mol.GetProp(prop))
                if smiles not in data[prop]:
                    data[prop][smiles] = []
                data[prop][smiles].append(score)
            else:  # make sure we don't miss anything important
                if prop not in skipped:
                    skipped.append(prop)
                    print "Skipping property '{}'".format(prop)
                continue

    # reconcile activity differences
    for dataset in dataset_names:
        for smiles, targets in data[dataset].items():
            targets = np.asarray(targets, dtype=int)
            if len(targets) == 1:
                data[dataset][smiles] = targets[0]
            elif np.all(targets == 0):
                data[dataset][smiles] = 0
            elif np.all(targets == 1):
                data[dataset][smiles] = 1
            else:
                if merge == 'max':
                    data[dataset][smiles] = max(targets)
                elif merge == 'min':
                    data[dataset][smiles] = min(targets)
                elif merge == 'majority-':  # 0.5 rounds down
                    data[dataset][smiles] = int(np.round(np.mean(targets)))
                elif merge == 'majority+':  # 0.5 rounds up
                    data[dataset][smiles] = (int(np.round(
                        np.mean(targets) + 1)) - 1)

    # save individual datasets
    for dataset in dataset_names:
        assert len(data[dataset])  # check for at least one molecule

        # build mol and target arrays
        mols = []
        targets = []
        for mol, target in data[dataset].items():
            mols.append(mol)
            targets.append(target)
        targets = np.asarray(targets, dtype=int)
        pos = np.count_nonzero(targets == 1)
        neg = np.count_nonzero(targets == 0)
        assert pos + neg == targets.size
        print '{}\t{}\t{}'.format(dataset, pos, neg)
        with gzip.open('{}.ism.gz'.format(dataset), 'wb') as f:
            for i, mol in enumerate(mols):
                f.write('{}\t{}\n'.format(mol, '{}-{}'.format(dataset, i)))
        with open('{}-targets.pkl'.format(dataset), 'wb') as f:
            cPickle.dump(targets, f, cPickle.HIGHEST_PROTOCOL)

if __name__ == '__main__':
    args = get_args()
    main(args.input, args.merge)
<|endoftext|>


<commit_before>from django.utils.encoding import smart_text

from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import BigInteger, SmallInteger, Boolean, Column, DateTime, Text
from sqlalchemy import ForeignKey, Integer, String
from sqlalchemy.ext.hybrid import hybrid_property, hybrid_method

import datetime
NOW = datetime.datetime.now()

ADDITION = 1
CHANGE = 2
DELETION = 3

Base = declarative_base()

class StudentMedicalContentType(Base):
    __tablename__ = 'cc_student_medical_content_type'

    id = Column(BigInteger, primary_key=True)
    name = Column(String)
    app_label = Column(String)
    model = Column(String)

    def __repr__(self):
        return self.model

class StudentMedicalLogEntry(Base):
    __tablename__ = 'cc_student_medical_log_entry'

    id = Column(BigInteger, primary_key=True)
    college_id = Column(Integer, nullable=False)
    action_time = Column(DateTime, default=NOW, nullable=False)
    content_type_id = Column(Integer)
    object_id = Column(Integer)
    object_repr = Column(String)
    action_flag = Column(SmallInteger)
    action_message = Column(Text)

    def __repr__(self):
        return smart_text(self.action_time)

    def __str__(self):
        if self.action_flag == ADDITION:
            return ('Added "%(object)s".') % {'object': self.object_repr}
        elif self.action_flag == CHANGE:
            return ('Changed "%(object)s" - %(changes)s') % {
                'object': self.object_repr,
                'changes': self.change_message,
            }
        elif self.action_flag == DELETION:
            return ('Deleted "%(object)s."') % {'object': self.object_repr}

        return ugettext('LogEntry Object')

    def is_addition(self):
        return self.action_flag == ADDITION

    def is_change(self):
        return self.action_flag == CHANGE

    def is_deletion(self):
        return self.action_flag == DELETION

class StudentMedicalManager(Base):
    __tablename__ = 'cc_student_medical_manager'

    # core
    id = Column(BigInteger, primary_key=True)
    college_id = Column(Integer, nullable=False)
    created_at = Column(DateTime, default=NOW, nullable=False)
    sitrep = Column(Boolean)
    athlete = Column(Boolean)
    sports = Column(String)
    emergency_contact = Column(Boolean)
    # forms and waivers
    cc_student_immunization = Column(Boolean)
    cc_student_medical_history = Column(Boolean)
    cc_student_health_insurance = Column(Boolean)
    cc_student_meni_waiver = Column(Boolean)
    cc_athlete_medical_history = Column(Boolean)
    cc_athlete_privacy_waiver = Column(Boolean)
    cc_athlete_reporting_waiver = Column(Boolean)
    cc_athlete_risk_waiver = Column(Boolean)
    cc_athlete_sicklecell_waiver = Column(Boolean)

    def __repr__(self):
        return str(self.college_id)

    #@hybrid_property
    @hybrid_method
    def current(self, day):
        """Is this the current manager for academic year?"""
        return self.created_at > day
    #current = property(_get_current)

# IDs must be unique pattern that does not repeat in any other
# item e.g 25 & 250 will not work.
SPORTS_MEN = (
    ("0","----Men's Sport----"),
    ("15","Baseball"),
    ("25","Basketball"),
    ("35","Cross Country"),
    ("45","Football"),
    ("55","Golf"),
    ("65","Lacrosse"),
    ("75","Soccer"),
    ("85","Swimming"),
    ("95","Tennis"),
    ("105","Track & Field"),
    ("120","Volleyball"),
)
SPORTS_WOMEN = (
    ("0","----Women's Sports----"),
    ("200","Basketball"),
    ("210","Cross Country"),
    ("220","Golf"),
    ("230","Lacrosse"),
    ("240","Soccer"),
    ("260","Softball"),
    ("270","Swimming"),
    ("280","Tennis"),
    ("290","Track & Field"),
    ("300","Volleyball"),
    ("305","Water Polo")
)

SPORTS = SPORTS_WOMEN + SPORTS_MEN
<commit_msg>added 'ice hockey' to the list of sports<commit_after>from django.utils.encoding import smart_text

from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import BigInteger, SmallInteger, Boolean, Column, DateTime, Text
from sqlalchemy import ForeignKey, Integer, String
from sqlalchemy.ext.hybrid import hybrid_property, hybrid_method

import datetime
NOW = datetime.datetime.now()

ADDITION = 1
CHANGE = 2
DELETION = 3

Base = declarative_base()

class StudentMedicalContentType(Base):
    __tablename__ = 'cc_student_medical_content_type'

    id = Column(BigInteger, primary_key=True)
    name = Column(String)
    app_label = Column(String)
    model = Column(String)

    def __repr__(self):
        return self.model

class StudentMedicalLogEntry(Base):
    __tablename__ = 'cc_student_medical_log_entry'

    id = Column(BigInteger, primary_key=True)
    college_id = Column(Integer, nullable=False)
    action_time = Column(DateTime, default=NOW, nullable=False)
    content_type_id = Column(Integer)
    object_id = Column(Integer)
    object_repr = Column(String)
    action_flag = Column(SmallInteger)
    action_message = Column(Text)

    def __repr__(self):
        return smart_text(self.action_time)

    def __str__(self):
        if self.action_flag == ADDITION:
            return ('Added "%(object)s".') % {'object': self.object_repr}
        elif self.action_flag == CHANGE:
            return ('Changed "%(object)s" - %(changes)s') % {
                'object': self.object_repr,
                'changes': self.change_message,
            }
        elif self.action_flag == DELETION:
            return ('Deleted "%(object)s."') % {'object': self.object_repr}

        return ugettext('LogEntry Object')

    def is_addition(self):
        return self.action_flag == ADDITION

    def is_change(self):
        return self.action_flag == CHANGE

    def is_deletion(self):
        return self.action_flag == DELETION

class StudentMedicalManager(Base):
    __tablename__ = 'cc_student_medical_manager'

    # core
    id = Column(BigInteger, primary_key=True)
    college_id = Column(Integer, nullable=False)
    created_at = Column(DateTime, default=NOW, nullable=False)
    sitrep = Column(Boolean)
    athlete = Column(Boolean)
    sports = Column(String)
    emergency_contact = Column(Boolean)
    # forms and waivers
    cc_student_immunization = Column(Boolean)
    cc_student_medical_history = Column(Boolean)
    cc_student_health_insurance = Column(Boolean)
    cc_student_meni_waiver = Column(Boolean)
    cc_athlete_medical_history = Column(Boolean)
    cc_athlete_privacy_waiver = Column(Boolean)
    cc_athlete_reporting_waiver = Column(Boolean)
    cc_athlete_risk_waiver = Column(Boolean)
    cc_athlete_sicklecell_waiver = Column(Boolean)

    def __repr__(self):
        return str(self.college_id)

    #@hybrid_property
    @hybrid_method
    def current(self, day):
        """Is this the current manager for academic year?"""
        return self.created_at > day
    #current = property(_get_current)

# IDs must be unique pattern that does not repeat in any other
# item e.g 25 & 250 will not work.
SPORTS_MEN = (
    ("0","----Men's Sport----"),
    ("15","Baseball"),
    ("25","Basketball"),
    ("35","Cross Country"),
    ("45","Football"),
    ("55","Golf"),
    ("60","Ice Hockey"),
    ("65","Lacrosse"),
    ("75","Soccer"),
    ("85","Swimming"),
    ("95","Tennis"),
    ("105","Track & Field"),
    ("120","Volleyball"),
)
SPORTS_WOMEN = (
    ("0","----Women's Sports----"),
    ("200","Basketball"),
    ("210","Cross Country"),
    ("220","Golf"),
    ("225","Ice Hockey"),
    ("230","Lacrosse"),
    ("240","Soccer"),
    ("260","Softball"),
    ("270","Swimming"),
    ("280","Tennis"),
    ("290","Track & Field"),
    ("300","Volleyball"),
    ("305","Water Polo")
)

SPORTS = SPORTS_WOMEN + SPORTS_MEN
<|endoftext|>


<commit_before>from django.contrib.syndication.views import Feed
from django.utils.feedgenerator import Rss201rev2Feed

class iTunesFeed(Rss201rev2Feed):
    def rss_attributes(self):
        attrs = super(iTunesFeed, self).rss_attributes()
        attrs['xmlns:itunes'] = 'http://www.itunes.com/dtds/podcast-1.0.dtd'
        return attrs

    def add_root_elements(self, handler):
        super(iTunesFeed, self).add_root_elements(handler)

        # Add owner details as iTunes fields
        if (self.feed['author_email'] is not None
                or self.feed['author_name'] is not None):
            handler.startElement(u'itunes:owner', {})

            if self.feed['author_email'] is not None:
                handler.addQuickElement(
                        u'itunes:email', self.feed['author_email'])

            if self.feed['author_name'] is not None:
                handler.addQuickElement(
                        u'itunes:name', self.feed['author_name'])

            handler.endElement(u'itunes:owner')

        # Need an iTunes author as well
        if self.feed['author_name'] is not None:
            handler.addQuickElement(
                u'itunes:author', self.feed['author_name'])

        # iTunes explicit
        if self.feed['itunes:explicit'] is not None:
            handler.addQuickElement(
                    u'itunes:explicit', self.feed['itunes:explicit'])

        # iTunes explicit
        if self.feed['itunes:image'] is not None:
            handler.addQuickElement(u'itunes:image', '', {
                'href': self.feed['itunes:image'],
            })

        # iTunes description/summary
        if self.feed['itunes:summary'] is not None:
            handler.addQuickElement(u'itunes:summary', self.feed['itunes:summary'])

        # iTunes Categories
        if self.feed['itunes:categories'] is not None:
            for i in self.feed['itunes:categories']:
                if isinstance(i, basestring):
                    # Single category
                    handler.addQuickElement(u'itunes:category', '', {
                        'text': i,
                    })
                else:
                    # Category with sub-category
                    for j in i:
                        handler.startElement(u'itunes:category', {
                            'text': j,
                        })
                    for j in i:
                        handler.endElement(u'itunes:category')

    def add_item_elements(self, handler, item):
        super(iTunesFeed, self).add_item_elements(handler, item)

        # File length in hours/mins/seconds
        if item['itunes:duration']:
            handler.addQuickElement(
                    u'itunes:duration', item['itunes:duration'])

class PodcastFeed(Feed):
    feed_type = iTunesFeed

    def item_extra_kwargs(self, item):
        return {
            'itunes:duration': self.item_itunes_duration(item),
        }

    def feed_extra_kwargs(self, obj):
        return {
            'itunes:explicit': self.itunes_explicit,
            'itunes:categories': self.itunes_categories,
            'itunes:summary': self.itunes_summary,
            'itunes:image': self.itunes_image,
        }
<commit_msg>Feed tidyups<commit_after>from django.contrib.syndication.views import Feed
from django.utils.feedgenerator import Rss201rev2Feed

class iTunesFeed(Rss201rev2Feed):
    def rss_attributes(self):
        attrs = super(iTunesFeed, self).rss_attributes()
        attrs['xmlns:itunes'] = 'http://www.itunes.com/dtds/podcast-1.0.dtd'
        return attrs

    def add_root_elements(self, handler):
        super(iTunesFeed, self).add_root_elements(handler)

        # Add owner details as iTunes fields
        if (self.feed['author_email'] is not None
                or self.feed['author_name'] is not None):
            handler.startElement(u'itunes:owner', {})

            if self.feed['author_email'] is not None:
                handler.addQuickElement(
                    u'itunes:email', self.feed['author_email'])

            if self.feed['author_name'] is not None:
                handler.addQuickElement(
                    u'itunes:name', self.feed['author_name'])

            handler.endElement(u'itunes:owner')

        # Need an iTunes author as well
        if self.feed['author_name'] is not None:
            handler.addQuickElement(
                u'itunes:author', self.feed['author_name'])

        # iTunes explicit
        if self.feed['itunes:explicit'] is not None:
            handler.addQuickElement(
                u'itunes:explicit', self.feed['itunes:explicit'])

        # iTunes image
        if self.feed['itunes:image'] is not None:
            handler.addQuickElement(u'itunes:image', '', {
                'href': self.feed['itunes:image'],
            })

        # iTunes description/summary
        if self.feed['itunes:summary'] is not None:
            handler.addQuickElement(u'itunes:summary', self.feed['itunes:summary'])

        # iTunes Categories
        if self.feed['itunes:categories'] is not None:
            for i in self.feed['itunes:categories']:
                if isinstance(i, basestring):
                    # Single category
                    handler.addQuickElement(u'itunes:category', '', {
                        'text': i,
                    })
                else:
                    # Category with sub-category
                    for j in i:
                        handler.startElement(u'itunes:category', {
                            'text': j,
                        })
                    for j in i:
                        handler.endElement(u'itunes:category')

    def add_item_elements(self, handler, item):
        super(iTunesFeed, self).add_item_elements(handler, item)

        # File length in hours/mins/seconds
        if item['itunes:duration']:
            handler.addQuickElement(
                u'itunes:duration', item['itunes:duration'])

class PodcastFeed(Feed):
    feed_type = iTunesFeed

    def item_extra_kwargs(self, item):
        return {
            'itunes:duration': self.item_itunes_duration(item),
        }

    def feed_extra_kwargs(self, obj):
        return {
            'itunes:explicit': self.itunes_explicit,
            'itunes:categories': self.itunes_categories,
            'itunes:summary': self.itunes_summary,
            'itunes:image': self.itunes_image,
        }
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2014 Pexego Sistemas Informáticos All Rights Reserved
#    $Jesús Ventosinos Mayor <jesus@pexego.es>$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp import models, fields, api, _, exceptions

class StockContainer(models.Model):

    _name = "stock.container"

    name = fields.Char("Container Ref.", required=True)
    date_expected = fields.Date("Date expected", required=True)
    move_ids = fields.One2many("stock.move", "container_id", "Moves",
                               readonly=True, copy=False)
    company_id = fields.\
        Many2one("res.company", "Company", required=True,
                 default=lambda self:
                 self.env['res.company'].
                 _company_default_get('stock.container'))

    _sql_constraints = [
        ('name_uniq', 'unique(name)', 'Container name must be unique')
    ]

    @api.multi
    def write(self, vals):
        if vals.get('date_expected', False):
            for container in self:
                if vals['date_expected'] != container.date_expected:
                    for move in container.move_ids:
                        move.date_expected = vals['date_expected']
        return super(StockContainer, self).write(vals)

class stock_picking(models.Model):

    _inherit = 'stock.picking'

    shipping_identifier = fields.Char('Shipping identifier', size=64)
    temp = fields.Boolean("Temp.")

    @api.multi
    def action_cancel(self):
        for pick in self:
            if pick.temp:
                for move in pick.move_lines:
                    if move.state == "assigned":
                        move.do_unreserve()
                    move.state = "draft"
                    move.picking_id = False
        return super(stock_picking, self).action_cancel()

class stock_move(models.Model):

    _inherit = 'stock.move'

    partner_id = fields.Many2one('res.partner', 'Partner')
    container_id = fields.Many2one('stock.container', "Container")
    subtotal_price = fields.Float('Subtotal', compute='_calc_subtotal')

    @api.multi
    def _calc_subtotal(self):
        for move in self:
            move.subtotal_price = move.price_unit * move.product_uom_qty

    @api.multi
    def write(self, vals):
        res = super(stock_move, self).write(vals)
        for move in self:
            move.refresh()
            if move.picking_type_id.code == 'incoming':
                if vals.get('date_expected', False):
                    self.env['stock.reservation'].\
                        reassign_reservation_dates(move.product_id)
            if vals.get('state', False) == 'assigned':
                reserv_ids = self.env["stock.reservation"].\
                    search([('move_id', '=', move.id),
                            ('sale_line_id', '!=', False)])
                if reserv_ids:
                    notify = True
                    for line in reserv_ids[0].sale_line_id.\
                            order_id.order_line:
                        if line.id != reserv_ids[0].sale_line_id.id:
                            for reserv in line.reservation_ids:
                                if reserv.state != 'assigned':
                                    notify = False
                    if notify:
                        sale = reserv_ids[0].sale_line_id.order_id
                        followers = sale.message_follower_ids
                        sale.message_post(body=_("The sale order is already assigned."),
                                          subtype='mt_comment',
                                          partner_ids=followers)
            if vals.get('container_id', False):
                container = self.env["stock.container"].\
                    browse(vals['container_id'])
                move.date_expected = container.date_expected
        return res

    @api.model
    def create(self, vals):
        res = super(stock_move, self).create(vals)
        if (vals.get('picking_type_id', False) and
                res.picking_type_id.code == 'incoming'):
            if 'date_expected' in vals.keys():
                self.env['stock.reservation'].\
                    reassign_reservation_dates(res.product_id)
        if not res.partner_id and res.picking_id.partner_id == \
                self.env.ref('purchase_picking.partner_multisupplier'):
            raise exceptions.Warning(
                _('Partner error'), _('Set the partner in the created moves'))
        return res

    def _get_master_data(self, cr, uid, move, company, context=None):
        partner, uid, currency = super(stock_move, self)._get_master_data(
            cr, uid, move, company, context)
        if move.picking_type_code == "incoming":
            if move.partner_id:
                partner = move.partner_id
            else:
                partner = move.picking_id.partner_id
        return partner, uid, currency

class stock_reservation(models.Model):
    _inherit = 'stock.reservation'

    @api.model
    def reassign_reservation_dates(self, product_id):
        uom_obj = self.env['product.uom']
        product_uom = product_id.uom_id
        reservations = self.search(
            [('product_id', '=', product_id.id),
             ('state', '=', 'confirmed')])
        moves = self.env['stock.move'].search(
            [('product_id', '=', product_id.id),
             ('state', '=', 'draft'),
             ('picking_type_id.code', '=', u'incoming')],
             order='date_expected')
        reservation_index = 0

        reservation_used = 0
        for move in moves:
            qty_used = 0
            product_uom_qty = uom_obj._compute_qty_obj(move.product_uom, move.product_uom_qty, product_uom)
            while qty_used < product_uom_qty and reservation_index < len(reservations):
                reservation = reservations[reservation_index]
                reservation_qty = reservation.product_uom_qty - reservation.reserved_availability
                reservation_qty = uom_obj._compute_qty_obj(reservation.product_uom, reservation_qty, product_uom)
                if reservation_qty - reservation_used <= product_uom_qty - qty_used:
                    reservation.date_planned = move.date_expected
                    reservation_index += 1
                    if reservation.sale_id:
                        sale = reservation.sale_id
                        followers = sale.message_follower_ids
                        sale.message_post(body=_("The date planned of the reservation was changed."),
                                          subtype='mt_comment',
                                          partner_ids=followers)
                else:
                    reservation_used += product_uom_qty - qty_used
                    break
                qty_used += reservation_qty - reservation_used
                reservation_used = 0
        while reservation_index < len(reservations):
            reservations[reservation_index].date_planned = False
            reservation_index += 1

    @api.multi
    def reassign(self):
        context = dict(self.env.context)
        context.pop('first', False)
        res = super(stock_reservation, self).reassign()
        for reservation in self:
            self.with_context(context).reassign_reservation_dates(reservation.product_id)
        return res
<commit_msg>[IMP] 'purchase_picking': Se ha hecho obligatorio el identificador de envio<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2014 Pexego Sistemas Informáticos All Rights Reserved
#    $Jesús Ventosinos Mayor <jesus@pexego.es>$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp import models, fields, api, _, exceptions

class StockContainer(models.Model):

    _name = "stock.container"

    name = fields.Char("Container Ref.", required=True)
    date_expected = fields.Date("Date expected", required=True)
    move_ids = fields.One2many("stock.move", "container_id", "Moves",
                               readonly=True, copy=False)

    user_id = fields.Many2one('Responsible', compute='_get_responsible')
    company_id = fields.\
        Many2one("res.company", "Company", required=True,
                 default=lambda self:
                 self.env['res.company'].
                 _company_default_get('stock.container'))

    _sql_constraints = [
        ('name_uniq', 'unique(name)', 'Container name must be unique')
    ]

    @api.one
    def _get_responsible(self):
        responsible = ''
        if self.picking_id:
            responsible = self.picking_id.commercial
        elif self.origin:
            responsible = self.env['sale.order'].search([('name', '=', self.origin)]).user_id
        return responsible

    @api.multi
    def write(self, vals):
        if vals.get('date_expected', False):
            for container in self:
                if vals['date_expected'] != container.date_expected:
                    for move in container.move_ids:
                        move.date_expected = vals['date_expected']
        return super(StockContainer, self).write(vals)

class stock_picking(models.Model):

    _inherit = 'stock.picking'

    shipping_identifier = fields.Char('Shipping identifier', required=True, size=64)
    temp = fields.Boolean("Temp.")

    @api.multi
    def action_cancel(self):
        for pick in self:
            if pick.temp:
                for move in pick.move_lines:
                    if move.state == "assigned":
                        move.do_unreserve()
                    move.state = "draft"
                    move.picking_id = False
        return super(stock_picking, self).action_cancel()

class stock_move(models.Model):

    _inherit = 'stock.move'

    partner_id = fields.Many2one('res.partner', 'Partner')
    container_id = fields.Many2one('stock.container', "Container")
    subtotal_price = fields.Float('Subtotal', compute='_calc_subtotal')

    @api.multi
    def _calc_subtotal(self):
        for move in self:
            move.subtotal_price = move.price_unit * move.product_uom_qty

    @api.multi
    def write(self, vals):
        res = super(stock_move, self).write(vals)
        for move in self:
            move.refresh()
            if move.picking_type_id.code == 'incoming':
                if vals.get('date_expected', False):
                    self.env['stock.reservation'].\
                        reassign_reservation_dates(move.product_id)
            if vals.get('state', False) == 'assigned':
                reserv_ids = self.env["stock.reservation"].\
                    search([('move_id', '=', move.id),
                            ('sale_line_id', '!=', False)])
                if reserv_ids:
                    notify = True
                    for line in reserv_ids[0].sale_line_id.\
                            order_id.order_line:
                        if line.id != reserv_ids[0].sale_line_id.id:
                            for reserv in line.reservation_ids:
                                if reserv.state != 'assigned':
                                    notify = False
                    if notify:
                        sale = reserv_ids[0].sale_line_id.order_id
                        followers = sale.message_follower_ids
                        sale.message_post(body=_("The sale order is already assigned."),
                                          subtype='mt_comment',
                                          partner_ids=followers)
            if vals.get('container_id', False):
                container = self.env["stock.container"].\
                    browse(vals['container_id'])
                move.date_expected = container.date_expected
        return res

    @api.model
    def create(self, vals):
        res = super(stock_move, self).create(vals)
        if (vals.get('picking_type_id', False) and
                res.picking_type_id.code == 'incoming'):
            if 'date_expected' in vals.keys():
                self.env['stock.reservation'].\
                    reassign_reservation_dates(res.product_id)
        if not res.partner_id and res.picking_id.partner_id == \
                self.env.ref('purchase_picking.partner_multisupplier'):
            raise exceptions.Warning(
                _('Partner error'), _('Set the partner in the created moves'))
        return res

    def _get_master_data(self, cr, uid, move, company, context=None):
        partner, uid, currency = super(stock_move, self)._get_master_data(
            cr, uid, move, company, context)
        if move.picking_type_code == "incoming":
            if move.partner_id:
                partner = move.partner_id
            else:
                partner = move.picking_id.partner_id
        return partner, uid, currency

class stock_reservation(models.Model):
    _inherit = 'stock.reservation'

    @api.model
    def reassign_reservation_dates(self, product_id):
        uom_obj = self.env['product.uom']
        product_uom = product_id.uom_id
        reservations = self.search(
            [('product_id', '=', product_id.id),
             ('state', '=', 'confirmed')])
        moves = self.env['stock.move'].search(
            [('product_id', '=', product_id.id),
             ('state', '=', 'draft'),
             ('picking_type_id.code', '=', u'incoming')],
             order='date_expected')
        reservation_index = 0

        reservation_used = 0
        for move in moves:
            qty_used = 0
            product_uom_qty = uom_obj._compute_qty_obj(move.product_uom, move.product_uom_qty, product_uom)
            while qty_used < product_uom_qty and reservation_index < len(reservations):
                reservation = reservations[reservation_index]
                reservation_qty = reservation.product_uom_qty - reservation.reserved_availability
                reservation_qty = uom_obj._compute_qty_obj(reservation.product_uom, reservation_qty, product_uom)
                if reservation_qty - reservation_used <= product_uom_qty - qty_used:
                    reservation.date_planned = move.date_expected
                    reservation_index += 1
                    if reservation.sale_id:
                        sale = reservation.sale_id
                        followers = sale.message_follower_ids
                        sale.message_post(body=_("The date planned of the reservation was changed."),
                                          subtype='mt_comment',
                                          partner_ids=followers)
                else:
                    reservation_used += product_uom_qty - qty_used
                    break
                qty_used += reservation_qty - reservation_used
                reservation_used = 0
        while reservation_index < len(reservations):
            reservations[reservation_index].date_planned = False
            reservation_index += 1

    @api.multi
    def reassign(self):
        context = dict(self.env.context)
        context.pop('first', False)
        res = super(stock_reservation, self).reassign()
        for reservation in self:
            self.with_context(context).reassign_reservation_dates(reservation.product_id)
        return res
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.utils.encoding import smart_text

from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import BigInteger, SmallInteger, Boolean, Column, DateTime, Text
from sqlalchemy import ForeignKey, Integer, String
from sqlalchemy.ext.hybrid import hybrid_property, hybrid_method

import datetime
NOW = datetime.datetime.now()

ADDITION = 1
CHANGE = 2
DELETION = 3

Base = declarative_base()

class StudentMedicalContentType(Base):
    __tablename__ = 'cc_student_medical_content_type'

    id = Column(BigInteger, primary_key=True)
    name = Column(String)
    app_label = Column(String)
    model = Column(String)

    def __repr__(self):
        return self.model

class StudentMedicalLogEntry(Base):
    __tablename__ = 'cc_student_medical_log_entry'

    id = Column(BigInteger, primary_key=True)
    college_id = Column(Integer, nullable=False)
    action_time = Column(DateTime, default=NOW, nullable=False)
    content_type_id = Column(Integer)
    object_id = Column(Integer)
    object_repr = Column(String)
    action_flag = Column(SmallInteger)
    action_message = Column(Text)

    def __repr__(self):
        return smart_text(self.action_time)

    def __str__(self):
        if self.action_flag == ADDITION:
            return ('Added "%(object)s".') % {'object': self.object_repr}
        elif self.action_flag == CHANGE:
            return ('Changed "%(object)s" - %(changes)s') % {
                'object': self.object_repr,
                'changes': self.change_message,
            }
        elif self.action_flag == DELETION:
            return ('Deleted "%(object)s."') % {'object': self.object_repr}

        return ugettext('LogEntry Object')

    def is_addition(self):
        return self.action_flag == ADDITION

    def is_change(self):
        return self.action_flag == CHANGE

    def is_deletion(self):
        return self.action_flag == DELETION

class StudentMedicalManager(Base):
    __tablename__ = 'cc_student_medical_manager'

    # core
    id = Column(BigInteger, primary_key=True)
    college_id = Column(Integer, nullable=False)
    created_at = Column(DateTime, default=NOW, nullable=False)
    sitrep = Column(Boolean)
    athlete = Column(Boolean)
    sports = Column(String)
    emergency_contact = Column(Boolean)
    # forms and waivers
    cc_student_immunization = Column(Boolean)
    cc_student_medical_history = Column(Boolean)
    cc_student_health_insurance = Column(Boolean)
    cc_student_meni_waiver = Column(Boolean)
    cc_athlete_medical_history = Column(Boolean)
    cc_athlete_privacy_waiver = Column(Boolean)
    cc_athlete_reporting_waiver = Column(Boolean)
    cc_athlete_risk_waiver = Column(Boolean)
    cc_athlete_sicklecell_waiver = Column(Boolean)
    staff_notes = Column(Text)

    def __repr__(self):
        return str(self.college_id)

    #@hybrid_property
    @hybrid_method
    def current(self, day):
        """Is this the current manager for academic year?"""
        return self.created_at > day
    #current = property(_get_current)

# IDs must be unique pattern that does not repeat in any other
# item e.g 25 & 250 will not work.
SPORTS_MEN = (
    ("0","----Men's Sport----"),
    ("15","Baseball"),
    ("25","Basketball"),
    ("35","Cross Country"),
    ("45","Football"),
    ("55","Golf"),
    ("61","Ice Hockey"),
    ("65","Lacrosse"),
    ("75","Soccer"),
    ("85","Swimming"),
    ("95","Tennis"),
    ("105","Track & Field"),
    ("120","Volleyball"),
)
SPORTS_WOMEN = (
    ("0","----Women's Sports----"),
    ("200","Basketball"),
    ("210","Cross Country"),
    ("220","Golf"),
    ("225","Ice Hockey"),
    ("230","Lacrosse"),
    ("240","Soccer"),
    ("260","Softball"),
    ("270","Swimming"),
    ("280","Tennis"),
    ("290","Track & Field"),
    ("300","Volleyball"),
    ("305","Water Polo")
)

SPORTS = SPORTS_WOMEN + SPORTS_MEN
<commit_msg>added physical_evaluation field to manager class<commit_after># -*- coding: utf-8 -*-
from django.utils.encoding import smart_text

from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import BigInteger, SmallInteger, Boolean, Column, DateTime, Text
from sqlalchemy import ForeignKey, Integer, String
from sqlalchemy.ext.hybrid import hybrid_property, hybrid_method

import datetime
NOW = datetime.datetime.now()

ADDITION = 1
CHANGE = 2
DELETION = 3

Base = declarative_base()

class StudentMedicalContentType(Base):
    __tablename__ = 'cc_student_medical_content_type'

    id = Column(BigInteger, primary_key=True)
    name = Column(String)
    app_label = Column(String)
    model = Column(String)

    def __repr__(self):
        return self.model

class StudentMedicalLogEntry(Base):
    __tablename__ = 'cc_student_medical_log_entry'

    id = Column(BigInteger, primary_key=True)
    college_id = Column(Integer, nullable=False)
    action_time = Column(DateTime, default=NOW, nullable=False)
    content_type_id = Column(Integer)
    object_id = Column(Integer)
    object_repr = Column(String)
    action_flag = Column(SmallInteger)
    action_message = Column(Text)

    def __repr__(self):
        return smart_text(self.action_time)

    def __str__(self):
        if self.action_flag == ADDITION:
            return ('Added "%(object)s".') % {'object': self.object_repr}
        elif self.action_flag == CHANGE:
            return ('Changed "%(object)s" - %(changes)s') % {
                'object': self.object_repr,
                'changes': self.change_message,
            }
        elif self.action_flag == DELETION:
            return ('Deleted "%(object)s."') % {'object': self.object_repr}

        return ugettext('LogEntry Object')

    def is_addition(self):
        return self.action_flag == ADDITION

    def is_change(self):
        return self.action_flag == CHANGE

    def is_deletion(self):
        return self.action_flag == DELETION

class StudentMedicalManager(Base):
    __tablename__ = 'cc_student_medical_manager'

    # core
    id = Column(BigInteger, primary_key=True)
    college_id = Column(Integer, nullable=False)
    created_at = Column(DateTime, default=NOW, nullable=False)
    sitrep = Column(Boolean)
    athlete = Column(Boolean)
    sports = Column(String)
    physical_evaluation = Column(String)
    emergency_contact = Column(Boolean)
    # forms and waivers
    cc_student_immunization = Column(Boolean)
    cc_student_medical_history = Column(Boolean)
    cc_student_health_insurance = Column(Boolean)
    cc_student_meni_waiver = Column(Boolean)
    cc_athlete_medical_history = Column(Boolean)
    cc_athlete_privacy_waiver = Column(Boolean)
    cc_athlete_reporting_waiver = Column(Boolean)
    cc_athlete_risk_waiver = Column(Boolean)
    cc_athlete_sicklecell_waiver = Column(Boolean)
    staff_notes = Column(Text)

    def __repr__(self):
        return str(self.college_id)

    #@hybrid_property
    @hybrid_method
    def current(self, day):
        """Is this the current manager for academic year?"""
        return self.created_at > day
    #current = property(_get_current)

# IDs must be unique pattern that does not repeat in any other
# item e.g 25 & 250 will not work.
SPORTS_MEN = (
    ("0","----Men's Sport----"),
    ("15","Baseball"),
    ("25","Basketball"),
    ("35","Cross Country"),
    ("45","Football"),
    ("55","Golf"),
    ("61","Ice Hockey"),
    ("65","Lacrosse"),
    ("75","Soccer"),
    ("85","Swimming"),
    ("95","Tennis"),
    ("105","Track & Field"),
    ("120","Volleyball"),
)
SPORTS_WOMEN = (
    ("0","----Women's Sports----"),
    ("200","Basketball"),
    ("210","Cross Country"),
    ("220","Golf"),
    ("225","Ice Hockey"),
    ("230","Lacrosse"),
    ("240","Soccer"),
    ("260","Softball"),
    ("270","Swimming"),
    ("280","Tennis"),
    ("290","Track & Field"),
    ("300","Volleyball"),
    ("305","Water Polo")
)

SPORTS = SPORTS_WOMEN + SPORTS_MEN
<|endoftext|>


<commit_before>
from .moveCard import to as moveTo # noqa
from .makeCard import ( # noqa
    makeCard,
    splitCard,
    mirrorCard,
    duplicateCard,
    reconnectRealBones,

    bipedSetup, # I want to get rid of this so bad!
)

from .build import ( # noqa
    buildRig,
    buildBones,
    deleteBones,
    )<commit_msg>Adding imports to card<commit_after>
from .moveCard import to as moveTo # noqa
from .makeCard import ( # noqa
    makeCard,
    splitCard,
    mirrorCard,
    duplicateCard,
    reconnectRealBones,
    cardIk,
    removeCardIk,
    getConnectors,
    getArrows,
    customUp,

    bipedSetup, # I want to get rid of this so bad
)

from .build import ( # noqa
    buildRig,
    buildBones,
    deleteBones,
    )
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

import pytest
pymongo = pytest.importorskip('pymongo')

from datetime import datetime
from toolz import pluck, reduceby, groupby

from blaze import into, compute, compute_up, discover, dshape

from blaze.compute.mongo import MongoQuery
from blaze.expr import symbol, by, floor, ceil
from blaze.compatibility import xfail

@pytest.fixture(scope='module')
def conn():
    try:
        return pymongo.MongoClient()
    except pymongo.errors.ConnectionFailure:
        pytest.skip('No mongo server running')

@pytest.fixture(scope='module')
def db(conn):
    return conn.test_db

bank_raw = [{'name': 'Alice', 'amount': 100},
            {'name': 'Alice', 'amount': 200},
            {'name': 'Bob', 'amount': 100},
            {'name': 'Bob', 'amount': 200},
            {'name': 'Bob', 'amount': 300}]

@pytest.yield_fixture
def big_bank(db):
    data = [{'name': 'Alice', 'amount': 100, 'city': 'New York City'},
            {'name': 'Alice', 'amount': 200, 'city': 'Austin'},
            {'name': 'Bob', 'amount': 100, 'city': 'New York City'},
            {'name': 'Bob', 'amount': 200, 'city': 'New York City'},
            {'name': 'Bob', 'amount': 300, 'city': 'San Francisco'}]
    coll = db.bigbank
    coll = into(coll, data)
    yield coll
    coll.drop()

@pytest.yield_fixture
def date_data(db):
    import numpy as np
    import pandas as pd
    n = 3
    d = {'name': ['Alice', 'Bob', 'Joe'],
         'when': [datetime(2010, 1, 1, i) for i in [1, 2, 3]],
         'amount': [100, 200, 300],
         'id': [1, 2, 3]}
    data = [dict(zip(d.keys(), [d[k][i] for k in d.keys()]))
            for i in range(n)]
    coll = into(db.date_data, data)
    yield coll
    coll.drop()

@pytest.yield_fixture
def bank(db):
    coll = db.bank
    coll = into(coll, bank_raw)
    yield coll
    coll.drop()

@pytest.yield_fixture
def missing_vals(db):
    data = [{'x': 1, 'z': 100},
            {'x': 2, 'y': 20, 'z': 200},
            {'x': 3, 'z': 300},
            {'x': 4, 'y': 40}]
    coll = db.missing_vals
    coll = into(coll, data)
    yield coll
    coll.drop()

@pytest.yield_fixture
def points(db):
    data = [{'x': 1, 'y': 10, 'z': 100},
            {'x': 2, 'y': 20, 'z': 200},
            {'x': 3, 'y': 30, 'z': 300},
            {'x': 4, 'y': 40, 'z': 400}]
    coll = db.points
    coll = into(coll, data)
    yield coll
    coll.drop()

@pytest.yield_fixture
def events(db):
    data = [{'time': datetime(2012, 1, 1, 12, 00, 00), 'x': 1},
            {'time': datetime(2012, 1, 2, 12, 00, 00), 'x': 2},
            {'time': datetime(2012, 1, 3, 12, 00, 00), 'x': 3}]
    coll = db.events
    coll = into(coll, data)
    yield coll
    coll.drop()

t = symbol('t', 'var * {name: string, amount: int}')
bigt = symbol('bigt', 'var * {name: string, amount: int, city: string}')
p = symbol('p', 'var * {x: int, y: int, z: int}')
e = symbol('e', 'var * {time: datetime, x: int}')

q = MongoQuery('fake', [])

def test_compute_on_db(bank, points):
    assert bank.database == points.database
    db = bank.database

    d = symbol(db.name, discover(db))
    assert (compute(d.points.x.sum(), db) ==
            sum(x['x'] for x in db.points.find()))

def test_symbol(bank):
    assert compute(t, bank) == list(pluck(['name', 'amount'], bank_raw))

def test_projection_one():
    assert compute_up(t[['name']], q).query == ({'$project': {'name': 1}},)

def test_head_one():
    assert compute_up(t.head(5), q).query == ({'$limit': 5},)

def test_head(bank):
    assert len(compute(t.head(2), bank)) == 2

def test_projection(bank):
    assert set(compute(t.name, bank)) == set(['Alice', 'Bob'])
    assert set(compute(t[['name']], bank)) == set([('Alice',), ('Bob',)])

def test_selection(bank):
    assert set(compute(t[t.name=='Alice'], bank)) == set([('Alice', 100),
                                                            ('Alice', 200)])
    assert set(compute(t['Alice'==t.name], bank)) == set([('Alice', 100),
                                                            ('Alice', 200)])
    assert set(compute(t[t.amount > 200], bank)) == set([('Bob', 300)])
    assert set(compute(t[t.amount >= 200], bank)) == set([('Bob', 300),
                                                          ('Bob', 200),
                                                          ('Alice', 200)])
    assert set(compute(t[t.name!='Alice'].name, bank)) == set(['Bob'])
    assert set(compute(t[(t.name=='Alice') & (t.amount > 150)], bank)) == \
            set([('Alice', 200)])
    assert set(compute(t[(t.name=='Alice') | (t.amount > 250)], bank)) == \
            set([('Alice', 200),
                    ('Alice', 100),
                    ('Bob', 300)])

def test_columnwise(points):
    assert set(compute(p.x + p.y, points)) == set([11, 22, 33, 44])

def test_columnwise_multiple_operands(points):
    expected = [x['x'] + x['y'] - x['z'] * x['x'] / 2 for x in points.find()]
    assert set(compute(p.x + p.y - p.z * p.x / 2, points)) == set(expected)

def test_arithmetic(points):
    expr = p.y // p.x
    assert set(compute(expr, points)) == set(compute(expr, points.find()))

def test_columnwise_mod(points):
    expected = [x['x'] % x['y'] - x['z'] * x['x'] / 2 + 1
                for x in points.find()]
    expr = p.x % p.y - p.z * p.x / 2 + 1
    assert set(compute(expr, points)) == set(expected)

@xfail(raises=NotImplementedError,
       reason='MongoDB does not implement certain arith ops')
def test_columnwise_pow(points):
    expected = [x['x'] ** x['y'] for x in points.find()]
    assert set(compute(p.x ** p.y, points)) == set(expected)

def test_by_one():
    assert compute_up(by(t.name, total=t.amount.sum()), q).query == \
            ({'$group': {'_id': {'name': '$name'},
                         'total': {'$sum': '$amount'}}},
             {'$project': {'total': '$total', 'name': '$_id.name'}})

def test_by(bank):
    assert set(compute(by(t.name, total=t.amount.sum()), bank)) == \
            set([('Alice', 300), ('Bob', 600)])
    assert set(compute(by(t.name, min=t.amount.min()), bank)) == \
            set([('Alice', 100), ('Bob', 100)])
    assert set(compute(by(t.name, max=t.amount.max()), bank)) == \
            set([('Alice', 200), ('Bob', 300)])
    assert set(compute(by(t.name, count=t.name.count()), bank)) == \
            set([('Alice', 2), ('Bob', 3)])

def test_reductions(bank):
    assert compute(t.amount.min(), bank) == 100
    assert compute(t.amount.max(), bank) == 300
    assert compute(t.amount.sum(), bank) == 900

def test_distinct(bank):
    assert set(compute(t.name.distinct(), bank)) == set(['Alice', 'Bob'])

def test_sort(bank):
    assert compute(t.amount.sort('amount'), bank) == \
            [100, 100, 200, 200, 300]
    assert compute(t.amount.sort('amount', ascending=False), bank) == \
            [300, 200, 200, 100, 100]

def test_by_multi_column(bank):
    assert set(compute(by(t[['name', 'amount']], count=t.count()), bank)) == \
            set([(d['name'], d['amount'], 1) for d in bank_raw])

def test_datetime_handling(events):
    assert set(compute(e[e.time >= datetime(2012, 1, 2, 12, 0, 0)].x,
                        events)) == set([2, 3])
    assert set(compute(e[e.time >= "2012-01-02"].x,
                        events)) == set([2, 3])

def test_summary_kwargs(bank):
    expr = by(t.name, total=t.amount.sum(), avg=t.amount.mean())
    result = compute(expr, bank)
    assert result == [('Bob', 200.0, 600), ('Alice', 150.0, 300)]

def test_summary_count(bank):
    expr = by(t.name, how_many=t.amount.count())
    result = compute(expr, bank)
    assert result == [('Bob', 3), ('Alice', 2)]

def test_summary_arith(bank):
    expr = by(t.name, add_one_and_sum=(t.amount + 1).sum())
    result = compute(expr, bank)
    assert result == [('Bob', 603), ('Alice', 302)]

def test_summary_arith_min(bank):
    expr = by(t.name, add_one_and_sum=(t.amount + 1).min())
    result = compute(expr, bank)
    assert result == [('Bob', 101), ('Alice', 101)]

def test_summary_arith_max(bank):
    expr = by(t.name, add_one_and_sum=(t.amount + 1).max())
    result = compute(expr, bank)
    assert result == [('Bob', 301), ('Alice', 201)]

def test_summary_complex_arith(bank):
    expr = by(t.name, arith=(100 - t.amount * 2 / 30.0).sum())
    result = compute(expr, bank)
    reducer = lambda acc, x: (100 - x['amount'] * 2 / 30.0) + acc
    expected = reduceby('name', reducer, bank.find(), 0)
    assert set(result) == set(expected.items())

def test_summary_complex_arith_multiple(bank):
    expr = by(t.name, arith=(100 - t.amount * 2 / 30.0).sum(),
              other=t.amount.mean())
    result = compute(expr, bank)
    reducer = lambda acc, x: (100 - x['amount'] * 2 / 30.0) + acc
    expected = reduceby('name', reducer, bank.find(), 0)

    mu = reduceby('name', lambda acc, x: acc + x['amount'], bank.find(), 0.0)
    values = list(mu.values())
    items = expected.items()
    counts = groupby('name', bank.find())
    items = [x + (float(v) / len(counts[x[0]]),) for x, v in zip(items, values)]
    assert set(result) == set(items)

def test_like(bank):
    bank.create_index([('name', pymongo.TEXT)])
    expr = t.like(name='*Alice*')
    result = compute(expr, bank)
    assert set(result) == set((('Alice', 100), ('Alice', 200)))

def test_like_multiple(big_bank):
    expr = bigt.like(name='*Bob*', city='*York*')
    result = compute(expr, big_bank)
    assert set(result) == set((('Bob', 100, 'New York City'),
                               ('Bob', 200, 'New York City')))

def test_like_mulitple_no_match(big_bank):
    # make sure we aren't OR-ing the matches
    expr = bigt.like(name='*York*', city='*Bob*')
    result = compute(expr, big_bank)
    assert not set(result)

def test_missing_values(missing_vals):
    assert discover(missing_vals).subshape[0] == \
            dshape('{x: int64, y: ?int64, z: ?int64}')

    assert set(compute(p.y, missing_vals)) == set([None, 20, None, 40])

def test_datetime_access(date_data):
    t = symbol('t',
            'var * {amount: float64, id: int64, name: string, when: datetime}')

    py_data = into(list, date_data) # a python version of the collection

    for attr in ['day', 'minute', 'second', 'year', 'month']:
        assert list(compute(getattr(t.when, attr), date_data)) == \
                list(compute(getattr(t.when, attr), py_data))

def test_datetime_access_and_arithmetic(date_data):
    t = symbol('t',
            'var * {amount: float64, id: int64, name: string, when: datetime}')

    py_data = into(list, date_data) # a python version of the collection

    expr = t.when.day + t.id

    assert list(compute(expr, date_data)) == list(compute(expr, py_data))

def test_floor_ceil(bank):
    t = symbol('t', discover(bank))
    assert set(compute(200 * floor(t.amount / 200), bank)) == set([0, 200])
    assert set(compute(200 * ceil(t.amount / 200), bank)) == set([200, 400])
<commit_msg>Test construction of Data<commit_after>from __future__ import absolute_import, division, print_function

import pytest
pymongo = pytest.importorskip('pymongo')

from datetime import datetime
from toolz import pluck, reduceby, groupby

from datashape import Record
from blaze import into, compute, compute_up, discover, dshape, Data

from blaze.compute.mongo import MongoQuery
from blaze.expr import symbol, by, floor, ceil
from blaze.compatibility import xfail

@pytest.fixture(scope='module')
def conn():
    try:
        return pymongo.MongoClient()
    except pymongo.errors.ConnectionFailure:
        pytest.skip('No mongo server running')

@pytest.fixture(scope='module')
def db(conn):
    return conn.test_db

bank_raw = [{'name': 'Alice', 'amount': 100},
            {'name': 'Alice', 'amount': 200},
            {'name': 'Bob', 'amount': 100},
            {'name': 'Bob', 'amount': 200},
            {'name': 'Bob', 'amount': 300}]

@pytest.yield_fixture
def big_bank(db):
    data = [{'name': 'Alice', 'amount': 100, 'city': 'New York City'},
            {'name': 'Alice', 'amount': 200, 'city': 'Austin'},
            {'name': 'Bob', 'amount': 100, 'city': 'New York City'},
            {'name': 'Bob', 'amount': 200, 'city': 'New York City'},
            {'name': 'Bob', 'amount': 300, 'city': 'San Francisco'}]
    coll = db.bigbank
    coll = into(coll, data)
    yield coll
    coll.drop()

@pytest.yield_fixture
def date_data(db):
    n = 3
    d = {'name': ['Alice', 'Bob', 'Joe'],
         'when': [datetime(2010, 1, 1, i) for i in [1, 2, 3]],
         'amount': [100, 200, 300],
         'id': [1, 2, 3]}
    data = [dict(zip(d.keys(), [d[k][i] for k in d.keys()]))
            for i in range(n)]
    coll = into(db.date_data, data)
    yield coll
    coll.drop()

@pytest.yield_fixture
def bank(db):
    coll = db.bank
    coll = into(coll, bank_raw)
    yield coll
    coll.drop()

@pytest.yield_fixture
def missing_vals(db):
    data = [{'x': 1, 'z': 100},
            {'x': 2, 'y': 20, 'z': 200},
            {'x': 3, 'z': 300},
            {'x': 4, 'y': 40}]
    coll = db.missing_vals
    coll = into(coll, data)
    yield coll
    coll.drop()

@pytest.yield_fixture
def points(db):
    data = [{'x': 1, 'y': 10, 'z': 100},
            {'x': 2, 'y': 20, 'z': 200},
            {'x': 3, 'y': 30, 'z': 300},
            {'x': 4, 'y': 40, 'z': 400}]
    coll = db.points
    coll = into(coll, data)
    yield coll
    coll.drop()

@pytest.yield_fixture
def events(db):
    data = [{'time': datetime(2012, 1, 1, 12, 00, 00), 'x': 1},
            {'time': datetime(2012, 1, 2, 12, 00, 00), 'x': 2},
            {'time': datetime(2012, 1, 3, 12, 00, 00), 'x': 3}]
    coll = db.events
    coll = into(coll, data)
    yield coll
    coll.drop()

t = symbol('t', 'var * {name: string, amount: int}')
bigt = symbol('bigt', 'var * {name: string, amount: int, city: string}')
p = symbol('p', 'var * {x: int, y: int, z: int}')
e = symbol('e', 'var * {time: datetime, x: int}')

q = MongoQuery('fake', [])

def test_compute_on_db(bank, points):
    assert bank.database == points.database
    db = bank.database

    d = symbol(db.name, discover(db))
    assert (compute(d.points.x.sum(), db) ==
            sum(x['x'] for x in db.points.find()))

def test_symbol(bank):
    assert compute(t, bank) == list(pluck(['name', 'amount'], bank_raw))

def test_projection_one():
    assert compute_up(t[['name']], q).query == ({'$project': {'name': 1}},)

def test_head_one():
    assert compute_up(t.head(5), q).query == ({'$limit': 5},)

def test_head(bank):
    assert len(compute(t.head(2), bank)) == 2

def test_projection(bank):
    assert set(compute(t.name, bank)) == set(['Alice', 'Bob'])
    assert set(compute(t[['name']], bank)) == set([('Alice',), ('Bob',)])

def test_selection(bank):
    assert set(compute(t[t.name=='Alice'], bank)) == set([('Alice', 100),
                                                            ('Alice', 200)])
    assert set(compute(t['Alice'==t.name], bank)) == set([('Alice', 100),
                                                            ('Alice', 200)])
    assert set(compute(t[t.amount > 200], bank)) == set([('Bob', 300)])
    assert set(compute(t[t.amount >= 200], bank)) == set([('Bob', 300),
                                                          ('Bob', 200),
                                                          ('Alice', 200)])
    assert set(compute(t[t.name!='Alice'].name, bank)) == set(['Bob'])
    assert set(compute(t[(t.name=='Alice') & (t.amount > 150)], bank)) == \
            set([('Alice', 200)])
    assert set(compute(t[(t.name=='Alice') | (t.amount > 250)], bank)) == \
            set([('Alice', 200),
                    ('Alice', 100),
                    ('Bob', 300)])

def test_columnwise(points):
    assert set(compute(p.x + p.y, points)) == set([11, 22, 33, 44])

def test_columnwise_multiple_operands(points):
    expected = [x['x'] + x['y'] - x['z'] * x['x'] / 2 for x in points.find()]
    assert set(compute(p.x + p.y - p.z * p.x / 2, points)) == set(expected)

def test_arithmetic(points):
    expr = p.y // p.x
    assert set(compute(expr, points)) == set(compute(expr, points.find()))

def test_columnwise_mod(points):
    expected = [x['x'] % x['y'] - x['z'] * x['x'] / 2 + 1
                for x in points.find()]
    expr = p.x % p.y - p.z * p.x / 2 + 1
    assert set(compute(expr, points)) == set(expected)

@xfail(raises=NotImplementedError,
       reason='MongoDB does not implement certain arith ops')
def test_columnwise_pow(points):
    expected = [x['x'] ** x['y'] for x in points.find()]
    assert set(compute(p.x ** p.y, points)) == set(expected)

def test_by_one():
    assert compute_up(by(t.name, total=t.amount.sum()), q).query == \
            ({'$group': {'_id': {'name': '$name'},
                         'total': {'$sum': '$amount'}}},
             {'$project': {'total': '$total', 'name': '$_id.name'}})

def test_by(bank):
    assert set(compute(by(t.name, total=t.amount.sum()), bank)) == \
            set([('Alice', 300), ('Bob', 600)])
    assert set(compute(by(t.name, min=t.amount.min()), bank)) == \
            set([('Alice', 100), ('Bob', 100)])
    assert set(compute(by(t.name, max=t.amount.max()), bank)) == \
            set([('Alice', 200), ('Bob', 300)])
    assert set(compute(by(t.name, count=t.name.count()), bank)) == \
            set([('Alice', 2), ('Bob', 3)])

def test_reductions(bank):
    assert compute(t.amount.min(), bank) == 100
    assert compute(t.amount.max(), bank) == 300
    assert compute(t.amount.sum(), bank) == 900

def test_distinct(bank):
    assert set(compute(t.name.distinct(), bank)) == set(['Alice', 'Bob'])

def test_sort(bank):
    assert compute(t.amount.sort('amount'), bank) == \
            [100, 100, 200, 200, 300]
    assert compute(t.amount.sort('amount', ascending=False), bank) == \
            [300, 200, 200, 100, 100]

def test_by_multi_column(bank):
    assert set(compute(by(t[['name', 'amount']], count=t.count()), bank)) == \
            set([(d['name'], d['amount'], 1) for d in bank_raw])

def test_datetime_handling(events):
    assert set(compute(e[e.time >= datetime(2012, 1, 2, 12, 0, 0)].x,
                        events)) == set([2, 3])
    assert set(compute(e[e.time >= "2012-01-02"].x,
                        events)) == set([2, 3])

def test_summary_kwargs(bank):
    expr = by(t.name, total=t.amount.sum(), avg=t.amount.mean())
    result = compute(expr, bank)
    assert result == [('Bob', 200.0, 600), ('Alice', 150.0, 300)]

def test_summary_count(bank):
    expr = by(t.name, how_many=t.amount.count())
    result = compute(expr, bank)
    assert result == [('Bob', 3), ('Alice', 2)]

def test_summary_arith(bank):
    expr = by(t.name, add_one_and_sum=(t.amount + 1).sum())
    result = compute(expr, bank)
    assert result == [('Bob', 603), ('Alice', 302)]

def test_summary_arith_min(bank):
    expr = by(t.name, add_one_and_sum=(t.amount + 1).min())
    result = compute(expr, bank)
    assert result == [('Bob', 101), ('Alice', 101)]

def test_summary_arith_max(bank):
    expr = by(t.name, add_one_and_sum=(t.amount + 1).max())
    result = compute(expr, bank)
    assert result == [('Bob', 301), ('Alice', 201)]

def test_summary_complex_arith(bank):
    expr = by(t.name, arith=(100 - t.amount * 2 / 30.0).sum())
    result = compute(expr, bank)
    reducer = lambda acc, x: (100 - x['amount'] * 2 / 30.0) + acc
    expected = reduceby('name', reducer, bank.find(), 0)
    assert set(result) == set(expected.items())

def test_summary_complex_arith_multiple(bank):
    expr = by(t.name, arith=(100 - t.amount * 2 / 30.0).sum(),
              other=t.amount.mean())
    result = compute(expr, bank)
    reducer = lambda acc, x: (100 - x['amount'] * 2 / 30.0) + acc
    expected = reduceby('name', reducer, bank.find(), 0)

    mu = reduceby('name', lambda acc, x: acc + x['amount'], bank.find(), 0.0)
    values = list(mu.values())
    items = expected.items()
    counts = groupby('name', bank.find())
    items = [x + (float(v) / len(counts[x[0]]),) for x, v in zip(items, values)]
    assert set(result) == set(items)

def test_like(bank):
    bank.create_index([('name', pymongo.TEXT)])
    expr = t.like(name='*Alice*')
    result = compute(expr, bank)
    assert set(result) == set((('Alice', 100), ('Alice', 200)))

def test_like_multiple(big_bank):
    expr = bigt.like(name='*Bob*', city='*York*')
    result = compute(expr, big_bank)
    assert set(result) == set((('Bob', 100, 'New York City'),
                               ('Bob', 200, 'New York City')))

def test_like_mulitple_no_match(big_bank):
    # make sure we aren't OR-ing the matches
    expr = bigt.like(name='*York*', city='*Bob*')
    result = compute(expr, big_bank)
    assert not set(result)

def test_missing_values(missing_vals):
    assert discover(missing_vals).subshape[0] == \
            dshape('{x: int64, y: ?int64, z: ?int64}')

    assert set(compute(p.y, missing_vals)) == set([None, 20, None, 40])

def test_datetime_access(date_data):
    t = symbol('t',
            'var * {amount: float64, id: int64, name: string, when: datetime}')

    py_data = into(list, date_data) # a python version of the collection

    for attr in ['day', 'minute', 'second', 'year', 'month']:
        assert list(compute(getattr(t.when, attr), date_data)) == \
                list(compute(getattr(t.when, attr), py_data))

def test_datetime_access_and_arithmetic(date_data):
    t = symbol('t',
            'var * {amount: float64, id: int64, name: string, when: datetime}')

    py_data = into(list, date_data) # a python version of the collection

    expr = t.when.day + t.id

    assert list(compute(expr, date_data)) == list(compute(expr, py_data))

def test_floor_ceil(bank):
    t = symbol('t', discover(bank))
    assert set(compute(200 * floor(t.amount / 200), bank)) == set([0, 200])
    assert set(compute(200 * ceil(t.amount / 200), bank)) == set([200, 400])

def test_Data_construct():
    d = Data('mongodb://localhost/test_db')
    assert isinstance(d.dshape.measure, Record)
<|endoftext|>