<commit_before># The MIT License (MIT)
# Copyright (c) 2016 Massachusetts Institute of Technology
#
# Authors: Victor Pankratius, Justin Li, Cody Rude
# This software has been created in projects supported by the US National
# Science Foundation and NASA (PI: Pankratius)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

# """@package GLDAS
# Provides classes for accessing GLDAS data.
# """

# mithagi required Base imports
from skdaccess.framework.data_class import DataFetcherStorage, TableWrapper
from skdaccess.utilities.grace_util import readTellusData, getStartEndDate

# Standard library imports
import os
from ftplib import FTP
import re
from collections import OrderedDict

# 3rd party package imports
import pandas as pd
import numpy as np

class DataFetcher(DataFetcherStorage):
    ''' Data Fetcher for GLDAS data '''

    def __init__(self, ap_paramList, start_date = None, end_date = None, resample = False):
        '''
        Construct a GLDAS Data Fetcher

        @param ap_paramList[geo_point]: Autolist of Geographic location tuples
        @param start_date: Beginning date
        @param end_date: Ending date
        @param resample: Resample the data to daily resolution, leaving NaN's in days without data (Default True)
        '''

        self.start_date = start_date
        self.end_date = end_date
        self.resample = resample
        super(DataFetcher, self).__init__(ap_paramList)

    def output(self):
        '''
        Create data wrapper of GLDAS data for specified geopoint.

        @return GLDAS Data Wrapper
        '''

        data_file = DataFetcher.getDataLocation('gldas')
        if data_file is None:
            print("No data available")
            return None

        geo_point_list = self.ap_paramList[0]()
        gldas_data_name = 'Equivalent Water Thickness (cm)'

        full_data, metadata = readTellusData(data_file, geo_point_list, 'Latitude','Longitude',
                                             'Water_Thickness', gldas_data_name, 'Time')

        # Get appropriate time range
        if self.start_date == None or self.end_date == None:
            start_date, end_date = getStartEndDate(full_data)

        if self.start_date != None:
            start_date = self.start_date

        elif type(self.start_date) == str:
            start_date = pd.to_datetime(self.start_date)

        if self.end_date != None:
            end_date = self.end_date

        elif type(self.end_date) == str:
            end_date == pd.to_datetime(self.end_date)

        for label in full_data.keys():

            full_data[label] = full_data[label][start_date:end_date]

            gldas_unc = pd.Series(np.ones(len(full_data[label]),dtype=np.float) * np.nan, index=full_data[label].index,name="Uncertainty")
            full_data[label] = pd.concat([full_data[label], gldas_unc], axis=1)

            if self.resample == True:
                full_data[label] = full_data[label].reindex(pd.date_range(start_date, end_date))

        return(TableWrapper(full_data, default_columns = ['Equivalent Water Thickness (cm)'],
                            default_error_columns=['Uncertainty']))

    @classmethod
    def downloadFullDataset(cls, out_file=None, use_file=None):
        '''
        Download GLDAS data

        @param out_file: Output filename for parsed data
        @param use_file: Directory of downloaded data. If None, data will be downloaded.

        @return Absolute path of parsed data
        '''

        # No post processing for this data is necessary. If local data is
        # specified, just set its location.
        if use_file != None:
            print('Setting data location for local data')
            return os.path.abspath(use_file)

        # If no local data, download data from server
        print("Downloading GLDAS Land Mass Data")
        ftp = FTP("podaac-ftp.jpl.nasa.gov")
        ftp.login()
        ftp.cwd('allData/tellus/L3/gldas_monthly/netcdf/')
        dir_list = list(ftp.nlst(''))
        file_list = [file for file in dir_list if re.search('.nc$', file)]
        if len(file_list) > 1:
            raise ValueError('Too many files found in GLDAS directory')

        if out_file == None:
            out_file = file_list[0]

        ftp.retrbinary('RETR ' + file_list[0], open(''+out_file, 'wb').write)

        cls.setDataLocation('gldas', os.path.abspath(file_list[0]))

    def __str__(self):
        '''
        String representation of data fetcher

        @return String listing the name and geopoint of data fetcher
        '''
        return 'GLDAS Data Fetcher' + super(DataFetcher, self).__str__()

<commit_msg>Updated for change in readTellusData<commit_after># The MIT License (MIT)
# Copyright (c) 2016 Massachusetts Institute of Technology
#
# Authors: Victor Pankratius, Justin Li, Cody Rude
# This software has been created in projects supported by the US National
# Science Foundation and NASA (PI: Pankratius)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

# """@package GLDAS
# Provides classes for accessing GLDAS data.
# """

# mithagi required Base imports
from skdaccess.framework.data_class import DataFetcherStorage, TableWrapper
from skdaccess.utilities.grace_util import readTellusData, getStartEndDate

# Standard library imports
import os
from ftplib import FTP
import re
from collections import OrderedDict

# 3rd party package imports
import pandas as pd
import numpy as np

class DataFetcher(DataFetcherStorage):
    ''' Data Fetcher for GLDAS data '''

    def __init__(self, ap_paramList, start_date = None, end_date = None, resample = False):
        '''
        Construct a GLDAS Data Fetcher

        @param ap_paramList[geo_point]: Autolist of Geographic location tuples
        @param start_date: Beginning date
        @param end_date: Ending date
        @param resample: Resample the data to daily resolution, leaving NaN's in days without data (Default True)
        '''

        self.start_date = start_date
        self.end_date = end_date
        self.resample = resample
        super(DataFetcher, self).__init__(ap_paramList)

    def output(self):
        '''
        Create data wrapper of GLDAS data for specified geopoint.

        @return GLDAS Data Wrapper
        '''

        data_file = DataFetcher.getDataLocation('gldas')
        if data_file is None:
            print("No data available")
            return None

        geo_point_list = self.ap_paramList[0]()
        gldas_data_name = 'Equivalent Water Thickness (cm)'

        full_data, metadata = readTellusData(data_file, geo_point_list, 'Latitude','Longitude',
                                             'Water_Thickness', gldas_data_name, 'Time')[:2]

        # Get appropriate time range
        if self.start_date == None or self.end_date == None:
            start_date, end_date = getStartEndDate(full_data)

        if self.start_date != None:
            start_date = self.start_date

        elif type(self.start_date) == str:
            start_date = pd.to_datetime(self.start_date)

        if self.end_date != None:
            end_date = self.end_date

        elif type(self.end_date) == str:
            end_date == pd.to_datetime(self.end_date)

        for label in full_data.keys():

            full_data[label] = full_data[label][start_date:end_date]

            gldas_unc = pd.Series(np.ones(len(full_data[label]),dtype=np.float) * np.nan, index=full_data[label].index,name="Uncertainty")
            full_data[label] = pd.concat([full_data[label], gldas_unc], axis=1)

            if self.resample == True:
                full_data[label] = full_data[label].reindex(pd.date_range(start_date, end_date))

        return(TableWrapper(full_data, default_columns = ['Equivalent Water Thickness (cm)'],
                            default_error_columns=['Uncertainty']))

    @classmethod
    def downloadFullDataset(cls, out_file=None, use_file=None):
        '''
        Download GLDAS data

        @param out_file: Output filename for parsed data
        @param use_file: Directory of downloaded data. If None, data will be downloaded.

        @return Absolute path of parsed data
        '''

        # No post processing for this data is necessary. If local data is
        # specified, just set its location.
        if use_file != None:
            print('Setting data location for local data')
            return os.path.abspath(use_file)

        # If no local data, download data from server
        print("Downloading GLDAS Land Mass Data")
        ftp = FTP("podaac-ftp.jpl.nasa.gov")
        ftp.login()
        ftp.cwd('allData/tellus/L3/gldas_monthly/netcdf/')
        dir_list = list(ftp.nlst(''))
        file_list = [file for file in dir_list if re.search('.nc$', file)]
        if len(file_list) > 1:
            raise ValueError('Too many files found in GLDAS directory')

        if out_file == None:
            out_file = file_list[0]

        ftp.retrbinary('RETR ' + file_list[0], open(''+out_file, 'wb').write)

        cls.setDataLocation('gldas', os.path.abspath(file_list[0]))

    def __str__(self):
        '''
        String representation of data fetcher

        @return String listing the name and geopoint of data fetcher
        '''
        return 'GLDAS Data Fetcher' + super(DataFetcher, self).__str__()

<|endoftext|>


<commit_before>from uricore import URI
from nose.tools import eq_
from uricore.core import uri_template
from collections import OrderedDict

colors = ["red", "green", "blue"]
punc = OrderedDict([('semi', ";"), ('dot', "."), ('comma', ",")])

def test_simple_string_expansion():
    yield eq_, "value", uri_template("{var}", var="value")
    yield eq_, "Hello%20World%21", uri_template("{hello}", hello="Hello World!")

def test_reserved_string_expansion():
    yield eq_, "value", uri_template("{+var}", var="value")
    yield eq_, "Hello%20World!", uri_template("{+hello}", hello="Hello World!")
    yield eq_, "/foo/bar/here", uri_template("{+path}/here", path="/foo/bar")

def test_fragment_expansion():
    yield eq_, "X#value", uri_template("X{#var}", var="value")
    yield eq_, "X#Hello%20World!", uri_template("X{#hello}", hello="Hello World!")

def test_string_expansion_multiple_vars():
    yield eq_, "map?1024,768", uri_template("map?{x,y}", x=1024, y=768)
    yield eq_, "1024,Hello%20World%21,768", uri_template("{x,hello,y}", x=1024, y=768, hello="Hello World!")

def test_reserved_expansion_multiple_vars():
    yield eq_, "1024,Hello%20World!,768", uri_template("{+x,hello,y}", x=1024, y=768, hello="Hello World!")
    yield eq_, "/foo/bar,1024/here", uri_template("{+path,x}/here", path="/foo/bar", x=1024)

def test_fragment_expansion_muliple_vars():
    yield eq_, "#1024,Hello%20World!,768", uri_template("{#x,hello,y}", x=1024, y=768, hello="Hello World!")
    yield eq_, "#/foo/bar,1024/here", uri_template("{#path,x}/here", path="/foo/bar", x=1024)

def test_label_expansion_dot_prefix():
    yield eq_, "X.1024.768", uri_template("X{.x,y}", x=1024, y=768)
    yield eq_, "X.value", uri_template("X{.var}", var="value")

def test_path_segments_slash_prefix():
    yield eq_, "/value", uri_template("{/var}", var="value")
    yield eq_, "/value/1024/here", uri_template("{/var,x}/here", var="value", x=1024)

def test_path_stype_params_semicolon_prefix():
    yield eq_, ";x=1024;y=768", uri_template("{;x,y}", x=1024, y=768)
    yield eq_, ";x=1024;y=768;empty", uri_template("{;x,y,empty}", x=1024, y=768, empty=None)

def test_form_style_and_prefix():
    yield eq_, "?x=1024&y=768", uri_template("{?x,y}", x=1024, y=768)
    yield eq_, "?x=1024&y=768&empty", uri_template("{?x,y,empty}", x=1024, y=768, empty=None)

def test_form_style_continuation():
    yield eq_, "?fixed=yes&x=1024", uri_template("?fixed=yes{&x}", x=1024)
    yield eq_, "&x=1024&y=768&empty", uri_template("{&x,y,empty}", x=1024, y=768, empty=None)

def test_string_expansion_with_value_mods():
    yield eq_, "val", uri_template("{var:3}", var="value")
    yield eq_, "value", uri_template("{var:30}", var="value")
    yield eq_, "red,green,blue", uri_template("{list}", list=colors)
    yield eq_, "red,green,blue", uri_template("{list*}", list=colors)
    yield eq_, "semi,%3B,dot,.,comma,%2C", uri_template("{keys}", keys=punc)
    yield eq_, "semi=%3B,dot=.,comma=%2C", uri_template("{keys*}", keys=punc)

def test_reserved_expansion_with_value_mods():
    yield eq_, "/foo/b/here", uri_template("{+path:6}/here", path="/foo/bar")
    yield eq_, "red,green,blue", uri_template("{+list}", list=colors)
    yield eq_, "red,green,blue", uri_template("{+list*}", list=colors)
    yield eq_, "semi,;,dot,.,comma,,", uri_template("{+keys}", keys=punc)
    yield eq_, "semi=;,dot=.,comma=,", uri_template("{+keys*}", keys=punc)

def test_fragment_expansion_with_value_mods():
    yield eq_, "#/foo/b/here", uri_template("{#path:6}/here", path="/foo/bar")
    yield eq_, "#red,green,blue", uri_template("{#list}", list=colors)
    yield eq_, "#red,green,blue", uri_template("{#list*}", list=colors)
    yield eq_, "#semi,;,dot,.,comma,,", uri_template("{#keys}", keys=punc)
    yield eq_, "#semi=;,dot=.,comma=,", uri_template("{#keys*}", keys=punc)

def test_label_expansion_with_value_mods():
    yield eq_, "X.val", uri_template("X{.var:3}", var="value")
    yield eq_, "X.red,green,blue", uri_template("X{.list}", list=colors)
    yield eq_, "X.red.green.blue", uri_template("X{.list*}", list=colors)
    yield eq_, "X.semi,%3B,dot,.,comma,%2C", uri_template("X{.keys}", keys=punc)
    yield eq_, "X.semi=%3B.dot=..comma=%2C", uri_template("X{.keys*}", keys=punc)

def test_path_expansion_with_value_mods():
    yield eq_, "/v/value", uri_template("{/var:1,var}", var="value")
    yield eq_, "/red,green,blue", uri_template("{/list}", list=colors)
    yield eq_, "/red/green/blue", uri_template("{/list*}", list=colors)
    yield eq_, "/red/green/blue/%2Ffoo", uri_template("{/list*,path:4}",
                                                    list=colors, path="/foo/bar")
    yield eq_, "/semi,%3B,dot,.,comma,%2C", uri_template("{/keys}", keys=punc)
    yield eq_, "/semi=%3B/dot=./comma=%2C", uri_template("{/keys*}", keys=punc)

def test_path_style_expansion_with_value_mods():
    yield eq_, ";hello=Hello", uri_template("{;hello:5}", hello="Hello World!")
    yield eq_, ";list=red,green,blue", uri_template("{;list}", list=colors)
    yield eq_, ";list=red;list=green;list=blue", uri_template("{;list*}", list=colors)
    yield eq_, ";keys=semi,%3B,dot,.,comma,%2C", uri_template("{;keys}", keys=punc)
    yield eq_, ";semi=%3B;dot=.;comma=%2C", uri_template("{;keys*}", keys=punc)

def test_form_expansion_with_value_mods():
    yield eq_, "?var=val", uri_template("{?var:3}", var="value")
    yield eq_, "?list=red,green,blue", uri_template("{?list}", list=colors)
    yield eq_, "?list=red&list=green&list=blue", uri_template("{?list*}", list=colors)
    yield eq_, "?keys=semi,%3B,dot,.,comma,%2C", uri_template("{?keys}", keys=punc)
    yield eq_, "?semi=%3B&dot=.&comma=%2C", uri_template("{?keys*}", keys=punc)

def test_form_continuation_expansion_with_value_mods():
    yield eq_, "&var=val", uri_template("{&var:3}", var="value")
    yield eq_, "&list=red,green,blue", uri_template("{&list}", list=colors)
    yield eq_, "&list=red&list=green&list=blue", uri_template("{&list*}", list=colors)
    yield eq_, "&keys=semi,%3B,dot,.,comma,%2C", uri_template("{&keys}", keys=punc)
    yield eq_, "&semi=%3B&dot=.&comma=%2C", uri_template("{&keys*}", keys=punc)

def test_uri_template():
    assert_equals(URI("http://example.com/value"),
                  URI.from_template("http://example.com/{var}", var="value"))
<commit_msg>Remove unneeded _eq definition<commit_after>from uricore import URI
from nose.tools import eq_
from uricore.core import uri_template
from collections import OrderedDict

colors = ["red", "green", "blue"]
punc = OrderedDict([('semi', ";"), ('dot', "."), ('comma', ",")])

def test_simple_string_expansion():
    yield eq_, "value", uri_template("{var}", var="value")
    yield eq_, "Hello%20World%21", uri_template("{hello}", hello="Hello World!")

def test_reserved_string_expansion():
    yield eq_, "value", uri_template("{+var}", var="value")
    yield eq_, "Hello%20World!", uri_template("{+hello}", hello="Hello World!")
    yield eq_, "/foo/bar/here", uri_template("{+path}/here", path="/foo/bar")

def test_fragment_expansion():
    yield eq_, "X#value", uri_template("X{#var}", var="value")
    yield eq_, "X#Hello%20World!", uri_template("X{#hello}", hello="Hello World!")

def test_string_expansion_multiple_vars():
    yield eq_, "map?1024,768", uri_template("map?{x,y}", x=1024, y=768)
    yield eq_, "1024,Hello%20World%21,768", uri_template("{x,hello,y}", x=1024, y=768, hello="Hello World!")

def test_reserved_expansion_multiple_vars():
    yield eq_, "1024,Hello%20World!,768", uri_template("{+x,hello,y}", x=1024, y=768, hello="Hello World!")
    yield eq_, "/foo/bar,1024/here", uri_template("{+path,x}/here", path="/foo/bar", x=1024)

def test_fragment_expansion_muliple_vars():
    yield eq_, "#1024,Hello%20World!,768", uri_template("{#x,hello,y}", x=1024, y=768, hello="Hello World!")
    yield eq_, "#/foo/bar,1024/here", uri_template("{#path,x}/here", path="/foo/bar", x=1024)

def test_label_expansion_dot_prefix():
    yield eq_, "X.1024.768", uri_template("X{.x,y}", x=1024, y=768)
    yield eq_, "X.value", uri_template("X{.var}", var="value")

def test_path_segments_slash_prefix():
    yield eq_, "/value", uri_template("{/var}", var="value")
    yield eq_, "/value/1024/here", uri_template("{/var,x}/here", var="value", x=1024)

def test_path_stype_params_semicolon_prefix():
    yield eq_, ";x=1024;y=768", uri_template("{;x,y}", x=1024, y=768)
    yield eq_, ";x=1024;y=768;empty", uri_template("{;x,y,empty}", x=1024, y=768, empty=None)

def test_form_style_and_prefix():
    yield eq_, "?x=1024&y=768", uri_template("{?x,y}", x=1024, y=768)
    yield eq_, "?x=1024&y=768&empty", uri_template("{?x,y,empty}", x=1024, y=768, empty=None)

def test_form_style_continuation():
    yield eq_, "?fixed=yes&x=1024", uri_template("?fixed=yes{&x}", x=1024)
    yield eq_, "&x=1024&y=768&empty", uri_template("{&x,y,empty}", x=1024, y=768, empty=None)

def test_string_expansion_with_value_mods():
    yield eq_, "val", uri_template("{var:3}", var="value")
    yield eq_, "value", uri_template("{var:30}", var="value")
    yield eq_, "red,green,blue", uri_template("{list}", list=colors)
    yield eq_, "red,green,blue", uri_template("{list*}", list=colors)
    yield eq_, "semi,%3B,dot,.,comma,%2C", uri_template("{keys}", keys=punc)
    yield eq_, "semi=%3B,dot=.,comma=%2C", uri_template("{keys*}", keys=punc)

def test_reserved_expansion_with_value_mods():
    yield eq_, "/foo/b/here", uri_template("{+path:6}/here", path="/foo/bar")
    yield eq_, "red,green,blue", uri_template("{+list}", list=colors)
    yield eq_, "red,green,blue", uri_template("{+list*}", list=colors)
    yield eq_, "semi,;,dot,.,comma,,", uri_template("{+keys}", keys=punc)
    yield eq_, "semi=;,dot=.,comma=,", uri_template("{+keys*}", keys=punc)

def test_fragment_expansion_with_value_mods():
    yield eq_, "#/foo/b/here", uri_template("{#path:6}/here", path="/foo/bar")
    yield eq_, "#red,green,blue", uri_template("{#list}", list=colors)
    yield eq_, "#red,green,blue", uri_template("{#list*}", list=colors)
    yield eq_, "#semi,;,dot,.,comma,,", uri_template("{#keys}", keys=punc)
    yield eq_, "#semi=;,dot=.,comma=,", uri_template("{#keys*}", keys=punc)

def test_label_expansion_with_value_mods():
    yield eq_, "X.val", uri_template("X{.var:3}", var="value")
    yield eq_, "X.red,green,blue", uri_template("X{.list}", list=colors)
    yield eq_, "X.red.green.blue", uri_template("X{.list*}", list=colors)
    yield eq_, "X.semi,%3B,dot,.,comma,%2C", uri_template("X{.keys}", keys=punc)
    yield eq_, "X.semi=%3B.dot=..comma=%2C", uri_template("X{.keys*}", keys=punc)

def test_path_expansion_with_value_mods():
    yield eq_, "/v/value", uri_template("{/var:1,var}", var="value")
    yield eq_, "/red,green,blue", uri_template("{/list}", list=colors)
    yield eq_, "/red/green/blue", uri_template("{/list*}", list=colors)
    yield eq_, "/red/green/blue/%2Ffoo", uri_template("{/list*,path:4}",
                                                    list=colors, path="/foo/bar")
    yield eq_, "/semi,%3B,dot,.,comma,%2C", uri_template("{/keys}", keys=punc)
    yield eq_, "/semi=%3B/dot=./comma=%2C", uri_template("{/keys*}", keys=punc)

def test_path_style_expansion_with_value_mods():
    yield eq_, ";hello=Hello", uri_template("{;hello:5}", hello="Hello World!")
    yield eq_, ";list=red,green,blue", uri_template("{;list}", list=colors)
    yield eq_, ";list=red;list=green;list=blue", uri_template("{;list*}", list=colors)
    yield eq_, ";keys=semi,%3B,dot,.,comma,%2C", uri_template("{;keys}", keys=punc)
    yield eq_, ";semi=%3B;dot=.;comma=%2C", uri_template("{;keys*}", keys=punc)

def test_form_expansion_with_value_mods():
    yield eq_, "?var=val", uri_template("{?var:3}", var="value")
    yield eq_, "?list=red,green,blue", uri_template("{?list}", list=colors)
    yield eq_, "?list=red&list=green&list=blue", uri_template("{?list*}", list=colors)
    yield eq_, "?keys=semi,%3B,dot,.,comma,%2C", uri_template("{?keys}", keys=punc)
    yield eq_, "?semi=%3B&dot=.&comma=%2C", uri_template("{?keys*}", keys=punc)

def test_form_continuation_expansion_with_value_mods():
    yield eq_, "&var=val", uri_template("{&var:3}", var="value")
    yield eq_, "&list=red,green,blue", uri_template("{&list}", list=colors)
    yield eq_, "&list=red&list=green&list=blue", uri_template("{&list*}", list=colors)
    yield eq_, "&keys=semi,%3B,dot,.,comma,%2C", uri_template("{&keys}", keys=punc)
    yield eq_, "&semi=%3B&dot=.&comma=%2C", uri_template("{&keys*}", keys=punc)

def test_uri_template():
    eq_(URI("http://example.com/value"),
        URI.from_template("http://example.com/{var}", var="value"))
<|endoftext|>


<commit_before>import datetime
from dateutil import tz
import requests
import os
import json

TRACKER_URL = "http://192.168.43.20:5000/api/tracker-sessions/create"
API_KEY = "fyu789gf75h46gs"

tzinfo = tz.tzoffset(None, datetime.timedelta(seconds=10800))

DATA_FILE = '/home/pi/meteo/src/dump.json'

def init_file():
    if not os.path.isfile(DATA_FILE):
        print "file initialized..."
        with open(DATA_FILE, mode='w', encoding='utf-8') as f:
            json.dump([], f)

def save_to_file(json_data):
    init_file()
    with open(DATA_FILE, mode='r', encoding='utf-8') as f:
        feeds = json.load(f)

    with open(DATA_FILE, mode='w', encoding='utf-8') as feedsjson:
        feeds.append(json_data)
        json.dump(feeds, feedsjson)

def post_internet(json_data):
    if os.path.isfile(DATA_FILE):
        print "data file exists"
        with open(DATA_FILE, mode='r', encoding='utf-8') as f:
            feeds = json.load(f)
        print "len is: %s" % len(feeds)
        for i, feed in enumerate(feeds):
            requests.post(TRACKER_URL, json=feed)
            print "posting... %s" % i
        os.remove(DATA_FILE)
    else:
        raise "no data file"
    requests.post(TRACKER_URL, json=json_data)
    print "internet posted!"

def post_update(latitude, longitude, timestamp):
    start_ts = end_ts = timestamp.replace(tzinfo=tzinfo)
    json_data = {
        'tracker_id': 3,
        'start_ts': str(start_ts),
        'end_ts': str(end_ts),
        'sessions': [{
            'latitude': latitude,
            'longitude': longitude
        }],
        'api_key': API_KEY
    }
    try:
        post_internet(json_data)
    except:
        save_to_file(json_data)
<commit_msg>dani fix<commit_after>import datetime
from dateutil import tz
import requests
import os
import json

TRACKER_URL = "http://192.168.43.20:5000/api/tracker-sessions/create"
API_KEY = "fyu789gf75h46gs"

tzinfo = tz.tzoffset(None, datetime.timedelta(seconds=10800))

DATA_FILE = '/home/pi/meteo/src/dump.json'

def init_file():
    if not os.path.isfile(DATA_FILE):
        print "file initialized..."
        with open(DATA_FILE, mode='w', encoding='utf-8') as f:
            json.dump([], f)

def save_to_file(json_data):
    init_file()
    with open(DATA_FILE, mode='r', encoding='utf-8') as f:
        feeds = json.load(f)

    with open(DATA_FILE, mode='w', encoding='utf-8') as feedsjson:
        feeds.append(json_data)
        json.dump(feeds, feedsjson)

def post_internet(json_data):
    if os.path.isfile(DATA_FILE):
        print "data file exists"
        with open(DATA_FILE, mode='r', encoding='utf-8') as f:
            feeds = json.load(f)
        print "len is: %s" % len(feeds)
        for i, feed in enumerate(feeds):
            requests.post(TRACKER_URL, json=feed)
            print "posting... %s" % i
        os.remove(DATA_FILE)
    requests.post(TRACKER_URL, json=json_data)
    print "internet posted!"

def post_update(latitude, longitude, timestamp):
    start_ts = end_ts = timestamp.replace(tzinfo=tzinfo)
    json_data = {
        'tracker_id': 3,
        'start_ts': str(start_ts),
        'end_ts': str(end_ts),
        'sessions': [{
            'latitude': latitude,
            'longitude': longitude
        }],
        'api_key': API_KEY
    }
    try:
        post_internet(json_data)
    except:
        save_to_file(json_data)
<|endoftext|>


<commit_before>from nose.tools import eq_
from unittest.mock import patch

from circle_asset.circle_api import get_latest_build, get_artifact_list
from circle_asset.project import Project

PROJECT = Project(username='pony',
                  project='bees',
                  api_root='http://example.com',
                  token=None)

class FakeResponse(object):
    def __init__(self, json_content):
        self._json = json_content

    def json(self):
        return self._json

@patch('requests.get')
def test_get_latest_build_pattern(rq_get):
    rq_get.configure_mock(return_value=FakeResponse([{'build_num': 10}]))
    build = get_latest_build(PROJECT, 'master')
    rq_get.assert_called_with('http://example.com/project/pony/bees/tree/master?limit=1&filter=successful', headers={'Accept': 'application/json'})
    eq_(build, 10)

@patch('requests.get')
def test_get_artifacts_pattern(rq_get):
    rq_get.configure_mock(return_value=FakeResponse([{'pretty_path': '$CIRCLE_ARTIFACTS/bees', 'url': 'http://example.com/bees'}]))
    artifacts = get_artifact_list(PROJECT, 12)
    rq_get.assert_called_with('http://example.com/project/pony/bees/12/artifacts', headers={'Accept': 'application/json'})
    eq_(artifacts, {'bees': 'http://example.com/bees'})

<commit_msg>Add test-cases for tokens in URLs<commit_after>from nose.tools import eq_
from unittest.mock import patch

from circle_asset.circle_api import get_latest_build, get_artifact_list
from circle_asset.project import Project

PROJECT = Project(username='pony',
                  project='bees',
                  api_root='http://example.com',
                  token=None)

PROJECT_TOKENED = PROJECT._replace(token='eyes')

class FakeResponse(object):
    def __init__(self, json_content):
        self._json = json_content

    def json(self):
        return self._json

@patch('requests.get')
def test_get_latest_build_pattern(rq_get):
    rq_get.configure_mock(return_value=FakeResponse([{'build_num': 10}]))
    build = get_latest_build(PROJECT, 'master')
    rq_get.assert_called_with('http://example.com/project/pony/bees/tree/master?limit=1&filter=successful', headers={'Accept': 'application/json'})
    eq_(build, 10)

@patch('requests.get')
def test_get_latest_build_token(rq_get):
    rq_get.configure_mock(return_value=FakeResponse([{'build_num': 10}]))
    build = get_latest_build(PROJECT_TOKENED, 'master')
    rq_get.assert_called_with('http://example.com/project/pony/bees/tree/master?limit=1&filter=successful&circle-token=eyes', headers={'Accept': 'application/json'})

@patch('requests.get')
def test_get_artifacts_pattern(rq_get):
    rq_get.configure_mock(return_value=FakeResponse([{'pretty_path': '$CIRCLE_ARTIFACTS/bees', 'url': 'http://example.com/bees'}]))
    artifacts = get_artifact_list(PROJECT, 12)
    rq_get.assert_called_with('http://example.com/project/pony/bees/12/artifacts', headers={'Accept': 'application/json'})
    eq_(artifacts, {'bees': 'http://example.com/bees'})

@patch('requests.get')
def test_get_artifacts_token(rq_get):
    rq_get.configure_mock(return_value=FakeResponse([{'pretty_path': '$CIRCLE_ARTIFACTS/bees', 'url': 'http://example.com/bees'}]))
    artifacts = get_artifact_list(PROJECT_TOKENED, 12)
    rq_get.assert_called_with('http://example.com/project/pony/bees/12/artifacts?circle-token=eyes', headers={'Accept': 'application/json'})
<|endoftext|>


<commit_before>import json
from stomp_message_controller import StompMessageController

class StompRegistrationController(StompMessageController):

	stomp_tasks = ["Register", "Unregister", "SendPeerList", "UserMessage", "MissingMethod"]

	def __init__(self):
		StompMessageController.__init__
		#print("StompRegistrationController.__init__()")
		self.registeredPeers = {}

	def run(self):
		print("StompRegistration.run()")

	def register(self, stomp_message):
		print("Register - stomp_registration_controller.register()")
		stomp_message.print_message()

		clientId = stomp_message.parsed_body['clientId']
		clientDesc = stomp_message.parsed_body['clientDesc']
		print("Registering Client Id - {0} - {1}".format(clientId, clientDesc))

		if (clientId not in self.registeredPeers.keys()):
			self.registeredPeers[clientId] = clientDesc
			self.broadcast_peer_list()

	def unregister(self, stomp_message):
		print("Unregister - stomp_registration_controller.unregister()")
		stomp_message.print_message()

		clientId = stomp_message.parsed_body['clientId']
		print("Unregistering Client Id - ", clientId)

		if (clientId in self.registeredPeers.keys()):
			del self.registeredPeers[clientId]
			self.broadcast_peer_list()

	def broadcast_peer_list(self):
		print("StompRegistrationController.broadcast_peer_list()")
		messageDict = {}
		messageDict["task"] = "PeerList"
		messageDict["registeredPeers"] = self.registeredPeers

		for clientId in self.registeredPeers.keys():
			messageDict["clientId"] = clientId
			print(clientId, messageDict)
			self.send_message(clientId, messageDict)

	def send_peer_list(self, stomp_message):
		print("SendPeerList - stomp_registration_controller.send_peer_list()")
		stomp_message.print_message()

		clientId = stomp_message.parsed_body['clientId']
		#print("Registered Client Ids - ", self.registeredClients)
		#print("will be sent to {0}".format(clientId))
		messageDict = {}
		messageDict["task"] = "PeerList"
		messageDict["clientId"] = clientId
		messageDict["registeredPeers"] = self.registeredPeers
		#print(messageDict)
		self.send_message(clientId, messageDict)

	def user_message(self, stomp_message):
		print("UserMessage - stomp_registration_controller.user_message()")
		stomp_message.print_message()

	def send_message(self, clientId, messageDict):
		#print("StompRegistrationController() send_message")
		jsonMsg = json.dumps(messageDict)
		messageQueue = "/queue/" + clientId
		#print(messageQueue)
		#print("Queue - {0}, Clients - {1}".format(messageQueue, self.registeredClients) )
		self.stompMessageBroker.sendMessage(jsonMsg, messageQueue)
<commit_msg>Removed .keys() from traversal of registeredPeers dictionary in stomp_registration_controller. Thx JEP<commit_after>import json
from stomp_message_controller import StompMessageController

class StompRegistrationController(StompMessageController):

	stomp_tasks = ["Register", "Unregister", "SendPeerList", "UserMessage", "MissingMethod"]

	def __init__(self):
		StompMessageController.__init__
		#print("StompRegistrationController.__init__()")
		self.registeredPeers = {}

	def run(self):
		print("StompRegistration.run()")

	def register(self, stomp_message):
		print("Register - stomp_registration_controller.register()")
		stomp_message.print_message()

		clientId = stomp_message.parsed_body['clientId']
		clientDesc = stomp_message.parsed_body['clientDesc']
		print("Registering Client Id - {0} - {1}".format(clientId, clientDesc))

		if (clientId not in self.registeredPeers):
			self.registeredPeers[clientId] = clientDesc
			self.broadcast_peer_list()

	def unregister(self, stomp_message):
		print("Unregister - stomp_registration_controller.unregister()")
		stomp_message.print_message()

		clientId = stomp_message.parsed_body['clientId']
		print("Unregistering Client Id - ", clientId)

		if (clientId in self.registeredPeers):
			del self.registeredPeers[clientId]
			self.broadcast_peer_list()

	def broadcast_peer_list(self):
		print("StompRegistrationController.broadcast_peer_list()")
		messageDict = {}
		messageDict["task"] = "PeerList"
		messageDict["registeredPeers"] = self.registeredPeers

		for clientId in self.registeredPeers:
			messageDict["clientId"] = clientId
			print(clientId, messageDict)
			self.send_message(clientId, messageDict)

	def send_peer_list(self, stomp_message):
		print("SendPeerList - stomp_registration_controller.send_peer_list()")
		stomp_message.print_message()

		clientId = stomp_message.parsed_body['clientId']
		#print("Registered Client Ids - ", self.registeredClients)
		#print("will be sent to {0}".format(clientId))
		messageDict = {}
		messageDict["task"] = "PeerList"
		messageDict["clientId"] = clientId
		messageDict["registeredPeers"] = self.registeredPeers
		#print(messageDict)
		self.send_message(clientId, messageDict)

	def user_message(self, stomp_message):
		print("UserMessage - stomp_registration_controller.user_message()")
		stomp_message.print_message()

	def send_message(self, clientId, messageDict):
		#print("StompRegistrationController() send_message")
		jsonMsg = json.dumps(messageDict)
		messageQueue = "/queue/" + clientId
		#print(messageQueue)
		#print("Queue - {0}, Clients - {1}".format(messageQueue, self.registeredClients) )
		self.stompMessageBroker.sendMessage(jsonMsg, messageQueue)
<|endoftext|>


<commit_before>"""This module implements a loader and dumper for the svmlight format

This format is a text-based format, with one sample per line. It does
not store zero valued features hence is suitable for sparse dataset.

The first element of each line can be used to store a target variable to
predict.

This format is used as the default format for both svmlight and the
libsvm command line programs.
"""

# Authors: Mathieu Blondel <mathieu@mblondel.org>
#          Lars Buitinck <L.J.Buitinck@uva.nl>
#          Olivier Grisel <olivier.grisel@ensta.org>
# License: Simple BSD.

from bz2 import BZ2File
from contextlib import closing
import gzip
import io
import os.path

import numpy as np
import scipy.sparse as sp

from ._svmlight_format import _load_svmlight_file
from .. import __version__
from ..utils import atleast2d_or_csr

def load_svmlight_file(f, n_features=None, dtype=np.float64,
                       multilabel=False, zero_based="auto", query_id=False):
    """Load datasets in the svmlight / libsvm format into sparse CSR matrix

    This format is a text-based format, with one sample per line. It does
    not store zero valued features hence is suitable for sparse dataset.

    The first element of each line can be used to store a target variable
    to predict.

    This format is used as the default format for both svmlight and the
    libsvm command line programs.

    Parsing a text based source can be expensive. When working on
    repeatedly on the same dataset, it is recommended to wrap this
    loader with joblib.Memory.cache to store a memmapped backup of the
    CSR results of the first call and benefit from the near instantaneous
    loading of memmapped structures for the subsequent calls.

    This implementation is naive: it does allocate too much memory and
    is slow since written in python. On large datasets it is recommended
    to use an optimized loader such as:

      https://github.com/mblondel/svmlight-loader

    In case the file contains a pairwise preference constraint (known
    as "qid" in the svmlight format) these are ignored unless the
    query_id parameter is set to True. These pairwise preference
    constraints can be used to contraint the combination of samples
    when using pairwise loss functions (as is the case in some
    learning to rank problems) so that only pairs with the same
    query_id value are considered.

    Parameters
    ----------
    f: {str, file-like, int}
        (Path to) a file to load. If a path ends in ".gz" or ".bz2", it will
        be uncompressed on the fly. If an integer is passed, it is assumed to
        be a file descriptor. A file-like or file descriptor will not be closed
        by this function. A file-like object must be opened in binary mode.

    n_features: int or None
        The number of features to use. If None, it will be inferred. This
        argument is useful to load several files that are subsets of a
        bigger sliced dataset: each subset might not have example of
        every feature, hence the inferred shape might vary from one
        slice to another.

    multilabel: boolean, optional
        Samples may have several labels each (see
        http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multilabel.html)

    zero_based: boolean or "auto", optional
        Whether column indices in f are zero-based (True) or one-based
        (False). If set to "auto", a heuristic check is applied to determine
        this from the file contents. Both kinds of files occur "in the wild",
        but they are unfortunately not self-identifying. Using "auto" or True
        should always be safe.

    query_id: boolean, defaults to False
        If True, will return the query_id array for each file.

    Returns
    -------
    X: scipy.sparse matrix of shape (n_samples, n_features)

    y: ndarray of shape (n_samples,), or, in the multilabel a list of
        tuples of length n_samples.

    query_id: array of shape (n_samples,)
       query_id for each sample. Only returned when query_id is set to
       True.

    See also
    --------
    load_svmlight_files: similar function for loading multiple files in this
    format, enforcing the same number of features/columns on all of them.
    """
    return tuple(load_svmlight_files([f], n_features, dtype, multilabel,
                                     zero_based, query_id))

def _gen_open(f):
    if isinstance(f, int):  # file descriptor
        return io.open(f, "rb", closefd=False)
    elif not isinstance(f, basestring):
        raise TypeError("expected {str, int, file-like}, got %s" % type(f))

    _, ext = os.path.splitext(f)
    if ext == ".gz":
        return gzip.open(f, "rb")
    elif ext == ".bz2":
        return BZ2File(f, "rb")
    else:
        return open(f, "rb")

def _open_and_load(f, dtype, multilabel, zero_based, query_id):
    if hasattr(f, "read"):
        return _load_svmlight_file(f, dtype, multilabel, zero_based, query_id)
    # XXX remove closing when Python 2.7+/3.1+ required
    with closing(_gen_open(f)) as f:
        return _load_svmlight_file(f, dtype, multilabel, zero_based, query_id)

def load_svmlight_files(files, n_features=None, dtype=np.float64,
                        multilabel=False, zero_based="auto", query_id=False):
    """Load dataset from multiple files in SVMlight format

    This function is equivalent to mapping load_svmlight_file over a list of
    files, except that the results are concatenated into a single, flat list
    and the samples vectors are constrained to all have the same number of
    features.

    In case the file contains a pairwise preference constraint (known
    as "qid" in the svmlight format) these are ignored unless the
    query_id parameter is set to True. These pairwise preference
    constraints can be used to contraint the combination of samples
    when using pairwise loss functions (as is the case in some
    learning to rank problems) so that only pairs with the same
    query_id value are considered.

    Parameters
    ----------
    files : iterable over {str, file-like, int}
        (Paths of) files to load. If a path ends in ".gz" or ".bz2", it will
        be uncompressed on the fly. If an integer is passed, it is assumed to
        be a file descriptor. File-likes and file descriptors will not be
        closed by this function. File-like objects must be opened in binary
        mode.

    n_features: int or None
        The number of features to use. If None, it will be inferred from the
        maximum column index occurring in any of the files.

    multilabel: boolean, optional
        Samples may have several labels each (see
        http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multilabel.html)

    zero_based: boolean or "auto", optional
        Whether column indices in files are zero-based (True) or one-based
        (False). If set to "auto", a heuristic check is applied to determine
        this from the files' contents. Both kinds of files occur "in the wild",
        but they are unfortunately not self-identifying. Using "auto" or True
        should always be safe.

    query_id: boolean, defaults to False
        If True, will return the query_id array for each file.

    Returns
    -------
    [X1, y1, ..., Xn, yn]
    where each (Xi, yi) pair is the result from load_svmlight_file(files[i]).

    If query_id is set to True, this will return instead [X1, y1, q1,
    ..., Xn, yn, qn] where (Xi, yi, qi) is the result from
    load_svmlight_file(files[i])

    Rationale
    ---------
    When fitting a model to a matrix X_train and evaluating it against a
    matrix X_test, it is essential that X_train and X_test have the same
    number of features (X_train.shape[1] == X_test.shape[1]). This may not
    be the case if you load the files individually with load_svmlight_file.

    See also
    --------
    load_svmlight_file
    """
    r = [_open_and_load(f, dtype, multilabel, bool(zero_based), bool(query_id))
         for f in files]

    if zero_based is False \
     or zero_based == "auto" and all(np.min(tmp[1]) > 0 for tmp in r):
        for ind in r:
            indices = ind[1]
            indices -= 1

    if n_features is None:
        n_features = max(ind[1].max() for ind in r) + 1

    result = []
    for data, indices, indptr, y, query_values in r:
        shape = (indptr.shape[0] - 1, n_features)
        result += sp.csr_matrix((data, indices, indptr), shape), y
        if query_id:
            result.append(query_values)

    return result

def _dump_svmlight(X, y, f, one_based, comment):
    is_sp = int(hasattr(X, "tocsr"))
    if X.dtype == np.float64:
        value_pattern = u"%d:%0.16e"
    else:
        value_pattern = u"%d:%f"

    if y.dtype.kind == 'i':
        line_pattern = u"%d %s\n"
    else:
        line_pattern = u"%f %s\n"

    f.write("# Generated by dump_svmlight_file from scikit-learn %s\n"
            % __version__)
    f.write("# Column indices are %s-based\n" % ["zero", "one"][one_based])

    if comment:
        f.write("#\n")
        f.writelines("# %s\n" % line for line in comment)

    for i in xrange(X.shape[0]):
        s = u" ".join([value_pattern % (j + one_based, X[i, j])
                       for j in X[i].nonzero()[is_sp]])
        f.write((line_pattern % (y[i], s)).encode('ascii'))

def dump_svmlight_file(X, y, f, zero_based=True, comment=None):
    """Dump the dataset in svmlight / libsvm file format.

    This format is a text-based format, with one sample per line. It does
    not store zero valued features hence is suitable for sparse dataset.

    The first element of each line can be used to store a target variable
    to predict.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape = [n_samples, n_features]
        Training vectors, where n_samples is the number of samples and
        n_features is the number of features.

    y : array-like, shape = [n_samples]
        Target values.

    f : string or file-like in binary mode
        If string, specifies the path that will contain the data.
        If file-like, data will be written to f. f should be opened in binary
        mode.

    zero_based : boolean, optional
        Whether column indices should be written zero-based (True) or one-based
        (False).

    comment : string, optional
        Comment to insert at the top of the file. This should be either a
        Unicode string, which will be encoded as UTF-8, or an ASCII byte
        string.
    """
    if comment is not None:
        # Convert comment string to list of lines in UTF-8.
        # If a byte string is passed, then check whether it's ASCII;
        # if a user wants to get fancy, they'll have to decode themselves.
        # Avoid mention of str and unicode types for Python 3.x compat.
        if isinstance(comment, bytes):
            comment.decode("ascii")     # just for the exception
        else:
            comment = comment.encode("utf-8")
        if "\0" in comment:
            raise ValueError("comment string contains NUL byte")
        comment = comment.splitlines()

    y = np.asarray(y)
    if y.ndim != 1:
        raise ValueError("expected y of shape [n_samples], got %r" % y)

    X = atleast2d_or_csr(X)
    if X.shape[0] != y.shape[0]:
        raise ValueError("X.shape[0] and y.shape[0] should be the same, "
                         "got: %r and %r instead." % (X.shape[0], y.shape[0]))

    one_based = not zero_based

    if hasattr(f, "write"):
        _dump_svmlight(X, y, f, one_based, comment)
    else:
        with open(f, "wb") as f:
            _dump_svmlight(X, y, f, one_based, comment)
<commit_msg>FIX: typo in ValueError message.<commit_after>"""This module implements a loader and dumper for the svmlight format

This format is a text-based format, with one sample per line. It does
not store zero valued features hence is suitable for sparse dataset.

The first element of each line can be used to store a target variable to
predict.

This format is used as the default format for both svmlight and the
libsvm command line programs.
"""

# Authors: Mathieu Blondel <mathieu@mblondel.org>
#          Lars Buitinck <L.J.Buitinck@uva.nl>
#          Olivier Grisel <olivier.grisel@ensta.org>
# License: Simple BSD.

from bz2 import BZ2File
from contextlib import closing
import gzip
import io
import os.path

import numpy as np
import scipy.sparse as sp

from ._svmlight_format import _load_svmlight_file
from .. import __version__
from ..utils import atleast2d_or_csr

def load_svmlight_file(f, n_features=None, dtype=np.float64,
                       multilabel=False, zero_based="auto", query_id=False):
    """Load datasets in the svmlight / libsvm format into sparse CSR matrix

    This format is a text-based format, with one sample per line. It does
    not store zero valued features hence is suitable for sparse dataset.

    The first element of each line can be used to store a target variable
    to predict.

    This format is used as the default format for both svmlight and the
    libsvm command line programs.

    Parsing a text based source can be expensive. When working on
    repeatedly on the same dataset, it is recommended to wrap this
    loader with joblib.Memory.cache to store a memmapped backup of the
    CSR results of the first call and benefit from the near instantaneous
    loading of memmapped structures for the subsequent calls.

    This implementation is naive: it does allocate too much memory and
    is slow since written in python. On large datasets it is recommended
    to use an optimized loader such as:

      https://github.com/mblondel/svmlight-loader

    In case the file contains a pairwise preference constraint (known
    as "qid" in the svmlight format) these are ignored unless the
    query_id parameter is set to True. These pairwise preference
    constraints can be used to contraint the combination of samples
    when using pairwise loss functions (as is the case in some
    learning to rank problems) so that only pairs with the same
    query_id value are considered.

    Parameters
    ----------
    f: {str, file-like, int}
        (Path to) a file to load. If a path ends in ".gz" or ".bz2", it will
        be uncompressed on the fly. If an integer is passed, it is assumed to
        be a file descriptor. A file-like or file descriptor will not be closed
        by this function. A file-like object must be opened in binary mode.

    n_features: int or None
        The number of features to use. If None, it will be inferred. This
        argument is useful to load several files that are subsets of a
        bigger sliced dataset: each subset might not have example of
        every feature, hence the inferred shape might vary from one
        slice to another.

    multilabel: boolean, optional
        Samples may have several labels each (see
        http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multilabel.html)

    zero_based: boolean or "auto", optional
        Whether column indices in f are zero-based (True) or one-based
        (False). If set to "auto", a heuristic check is applied to determine
        this from the file contents. Both kinds of files occur "in the wild",
        but they are unfortunately not self-identifying. Using "auto" or True
        should always be safe.

    query_id: boolean, defaults to False
        If True, will return the query_id array for each file.

    Returns
    -------
    X: scipy.sparse matrix of shape (n_samples, n_features)

    y: ndarray of shape (n_samples,), or, in the multilabel a list of
        tuples of length n_samples.

    query_id: array of shape (n_samples,)
       query_id for each sample. Only returned when query_id is set to
       True.

    See also
    --------
    load_svmlight_files: similar function for loading multiple files in this
    format, enforcing the same number of features/columns on all of them.
    """
    return tuple(load_svmlight_files([f], n_features, dtype, multilabel,
                                     zero_based, query_id))

def _gen_open(f):
    if isinstance(f, int):  # file descriptor
        return io.open(f, "rb", closefd=False)
    elif not isinstance(f, basestring):
        raise TypeError("expected {str, int, file-like}, got %s" % type(f))

    _, ext = os.path.splitext(f)
    if ext == ".gz":
        return gzip.open(f, "rb")
    elif ext == ".bz2":
        return BZ2File(f, "rb")
    else:
        return open(f, "rb")

def _open_and_load(f, dtype, multilabel, zero_based, query_id):
    if hasattr(f, "read"):
        return _load_svmlight_file(f, dtype, multilabel, zero_based, query_id)
    # XXX remove closing when Python 2.7+/3.1+ required
    with closing(_gen_open(f)) as f:
        return _load_svmlight_file(f, dtype, multilabel, zero_based, query_id)

def load_svmlight_files(files, n_features=None, dtype=np.float64,
                        multilabel=False, zero_based="auto", query_id=False):
    """Load dataset from multiple files in SVMlight format

    This function is equivalent to mapping load_svmlight_file over a list of
    files, except that the results are concatenated into a single, flat list
    and the samples vectors are constrained to all have the same number of
    features.

    In case the file contains a pairwise preference constraint (known
    as "qid" in the svmlight format) these are ignored unless the
    query_id parameter is set to True. These pairwise preference
    constraints can be used to contraint the combination of samples
    when using pairwise loss functions (as is the case in some
    learning to rank problems) so that only pairs with the same
    query_id value are considered.

    Parameters
    ----------
    files : iterable over {str, file-like, int}
        (Paths of) files to load. If a path ends in ".gz" or ".bz2", it will
        be uncompressed on the fly. If an integer is passed, it is assumed to
        be a file descriptor. File-likes and file descriptors will not be
        closed by this function. File-like objects must be opened in binary
        mode.

    n_features: int or None
        The number of features to use. If None, it will be inferred from the
        maximum column index occurring in any of the files.

    multilabel: boolean, optional
        Samples may have several labels each (see
        http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multilabel.html)

    zero_based: boolean or "auto", optional
        Whether column indices in files are zero-based (True) or one-based
        (False). If set to "auto", a heuristic check is applied to determine
        this from the files' contents. Both kinds of files occur "in the wild",
        but they are unfortunately not self-identifying. Using "auto" or True
        should always be safe.

    query_id: boolean, defaults to False
        If True, will return the query_id array for each file.

    Returns
    -------
    [X1, y1, ..., Xn, yn]
    where each (Xi, yi) pair is the result from load_svmlight_file(files[i]).

    If query_id is set to True, this will return instead [X1, y1, q1,
    ..., Xn, yn, qn] where (Xi, yi, qi) is the result from
    load_svmlight_file(files[i])

    Rationale
    ---------
    When fitting a model to a matrix X_train and evaluating it against a
    matrix X_test, it is essential that X_train and X_test have the same
    number of features (X_train.shape[1] == X_test.shape[1]). This may not
    be the case if you load the files individually with load_svmlight_file.

    See also
    --------
    load_svmlight_file
    """
    r = [_open_and_load(f, dtype, multilabel, bool(zero_based), bool(query_id))
         for f in files]

    if zero_based is False \
     or zero_based == "auto" and all(np.min(tmp[1]) > 0 for tmp in r):
        for ind in r:
            indices = ind[1]
            indices -= 1

    if n_features is None:
        n_features = max(ind[1].max() for ind in r) + 1

    result = []
    for data, indices, indptr, y, query_values in r:
        shape = (indptr.shape[0] - 1, n_features)
        result += sp.csr_matrix((data, indices, indptr), shape), y
        if query_id:
            result.append(query_values)

    return result

def _dump_svmlight(X, y, f, one_based, comment):
    is_sp = int(hasattr(X, "tocsr"))
    if X.dtype == np.float64:
        value_pattern = u"%d:%0.16e"
    else:
        value_pattern = u"%d:%f"

    if y.dtype.kind == 'i':
        line_pattern = u"%d %s\n"
    else:
        line_pattern = u"%f %s\n"

    f.write("# Generated by dump_svmlight_file from scikit-learn %s\n"
            % __version__)
    f.write("# Column indices are %s-based\n" % ["zero", "one"][one_based])

    if comment:
        f.write("#\n")
        f.writelines("# %s\n" % line for line in comment)

    for i in xrange(X.shape[0]):
        s = u" ".join([value_pattern % (j + one_based, X[i, j])
                       for j in X[i].nonzero()[is_sp]])
        f.write((line_pattern % (y[i], s)).encode('ascii'))

def dump_svmlight_file(X, y, f, zero_based=True, comment=None):
    """Dump the dataset in svmlight / libsvm file format.

    This format is a text-based format, with one sample per line. It does
    not store zero valued features hence is suitable for sparse dataset.

    The first element of each line can be used to store a target variable
    to predict.

    Parameters
    ----------
    X : {array-like, sparse matrix}, shape = [n_samples, n_features]
        Training vectors, where n_samples is the number of samples and
        n_features is the number of features.

    y : array-like, shape = [n_samples]
        Target values.

    f : string or file-like in binary mode
        If string, specifies the path that will contain the data.
        If file-like, data will be written to f. f should be opened in binary
        mode.

    zero_based : boolean, optional
        Whether column indices should be written zero-based (True) or one-based
        (False).

    comment : string, optional
        Comment to insert at the top of the file. This should be either a
        Unicode string, which will be encoded as UTF-8, or an ASCII byte
        string.
    """
    if comment is not None:
        # Convert comment string to list of lines in UTF-8.
        # If a byte string is passed, then check whether it's ASCII;
        # if a user wants to get fancy, they'll have to decode themselves.
        # Avoid mention of str and unicode types for Python 3.x compat.
        if isinstance(comment, bytes):
            comment.decode("ascii")     # just for the exception
        else:
            comment = comment.encode("utf-8")
        if "\0" in comment:
            raise ValueError("comment string contains NUL byte")
        comment = comment.splitlines()

    y = np.asarray(y)
    if y.ndim != 1:
        raise ValueError("expected y of shape (n_samples,), got %r" % (y.shape,))

    X = atleast2d_or_csr(X)
    if X.shape[0] != y.shape[0]:
        raise ValueError("X.shape[0] and y.shape[0] should be the same, "
                         "got: %r and %r instead." % (X.shape[0], y.shape[0]))

    one_based = not zero_based

    if hasattr(f, "write"):
        _dump_svmlight(X, y, f, one_based, comment)
    else:
        with open(f, "wb") as f:
            _dump_svmlight(X, y, f, one_based, comment)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import unittest
import os
import swc2vtk

class TestVtkGenerator(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        pass

    @classmethod
    def tearDownClass(cls):
        pass

    def setUp(self):
        self.swc_file_path = os.path.join('tests', 'simple.swc')
        self.data_file_path = os.path.join('tests', 'simple.dat')
        self.output_file_path = os.path.join('output.vtk')
        self.vtk_generator = swc2vtk.VtkGenerator()

    def tearDown(self):
        self.vtk_generator.write_vtk(self.output_file_path)

    def test_add_cylinder(self):
        self.vtk_generator.add_cylinder()

    def test_add_swc(self):
        simple_cmp_size = 11
        self.vtk_generator.add_swc(self.swc_file_path)
        self.assertEqual(simple_cmp_size, len(self.vtk_generator.swc_list[0].data))

    def test_add_swc_flip(self):
        simple_cmp_size = 11
        self.vtk_generator.add_swc(self.swc_file_path, inv_x=True, inv_y=True, inv_z=True,
                                   shift_x=100, shift_y=100, shift_z=100)
        self.assertEqual(simple_cmp_size, len(self.vtk_generator.swc_list[0].data))

    def test_add_datafile(self):
        simple_cmp_size = 11
        self.vtk_generator.add_swc(self.swc_file_path)
        self.vtk_generator.add_datafile(self.data_file_path)
        self.assertEqual(simple_cmp_size, len(self.vtk_generator.swc_list[0].data))

    def test_draw_mode1(self):
        self.vtk_generator.set_draw_mode(1)
        self.vtk_generator.add_swc(self.swc_file_path)

    def test_draw_mode2(self):
        self.vtk_generator.set_draw_mode(2)
        self.vtk_generator.add_swc(self.swc_file_path)

    def test_draw_mode3(self):
        self.vtk_generator.set_draw_mode(3)
        self.vtk_generator.add_swc(self.swc_file_path)

    def test_draw_mode4(self):
        self.vtk_generator.set_draw_mode(4)
        self.vtk_generator.add_swc(self.swc_file_path)

    def test_write_vtk_options(self):
        self.vtk_generator.add_swc(self.swc_file_path)
        self.vtk_generator.write_vtk(self.output_file_path, fixedval=True, movingval=True, coloring=True,
                                     diam_ratio=0.2, normalize_diam=True, radius_data=True, type_data=True)

    def test_write_volume_vtk(self):
        self.vtk_generator.add_swc(self.swc_file_path)
        self.vtk_generator.write_volume_vtk('simple.vtk', origin=(-10.0, -10.0, -10.0), ratio=(1, 1, 1), div=(20, 20, 20))\

    def test_add_mark(self):
        self.vtk_generator.add_mark()

    def test_add_connection(self):
        self.vtk_generator.add_connection(0, 0, 100, 200)

def suite():
    suite = unittest.TestSuite()
    suite.addTests(unittest.makeSuite(TestVtkGenerator))
    return suite

if __name__ == '__main__':
    unittest.main()
<commit_msg>fix test bug<commit_after># -*- coding: utf-8 -*-
import unittest
import os
import swc2vtk

class TestVtkGenerator(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        pass

    @classmethod
    def tearDownClass(cls):
        pass

    def setUp(self):
        self.swc_file_path = os.path.join('tests', 'simple.swc')
        self.data_file_path = os.path.join('tests', 'simple.dat')
        self.output_file_path = os.path.join('output.vtk')
        self.vtk_generator = swc2vtk.VtkGenerator()

    def tearDown(self):
        self.vtk_generator.write_vtk(self.output_file_path)

    def test_add_cylinder(self):
        self.vtk_generator.add_cylinder()

    def test_add_swc(self):
        simple_cmp_size = 11
        self.vtk_generator.add_swc(self.swc_file_path)
        self.assertEqual(simple_cmp_size, len(self.vtk_generator.swc_list[0].data))

    def test_add_swc_flip(self):
        simple_cmp_size = 11
        self.vtk_generator.add_swc(self.swc_file_path, inv_x=True, inv_y=True, inv_z=True,
                                   shift_x=100, shift_y=100, shift_z=100)
        self.assertEqual(simple_cmp_size, len(self.vtk_generator.swc_list[0].data))

    def test_add_datafile(self):
        simple_cmp_size = 11
        self.vtk_generator.add_swc(self.swc_file_path)
        self.vtk_generator.add_datafile(self.data_file_path)
        self.assertEqual(simple_cmp_size, len(self.vtk_generator.swc_list[0].data))

    def test_draw_mode1(self):
        self.vtk_generator.set_draw_mode(1)
        self.vtk_generator.add_swc(self.swc_file_path)

    def test_draw_mode2(self):
        self.vtk_generator.set_draw_mode(2)
        self.vtk_generator.add_swc(self.swc_file_path)

    def test_draw_mode3(self):
        self.vtk_generator.set_draw_mode(3)
        self.vtk_generator.add_swc(self.swc_file_path)

    def test_draw_mode4(self):
        self.vtk_generator.set_draw_mode(4)
        self.vtk_generator.add_swc(self.swc_file_path)

    def test_write_vtk_options(self):
        self.vtk_generator.add_swc(self.swc_file_path)
        self.vtk_generator.write_vtk(self.output_file_path, fixedval=True, movingval=True, coloring=True,
                                     diam_ratio=0.2, normalize_diam=True, radius_data=True, type_data=True)

    def test_write_volume_vtk(self):
        self.vtk_generator.add_swc(self.swc_file_path)
        self.vtk_generator.write_volume_vtk('simple.vtk', origin=(-10.0, -10.0, -10.0), ratio=(1, 1, 1), div=(20, 20, 20))\

    def test_add_mark(self):
        self.vtk_generator.add_swc(self.swc_file_path)
        self.vtk_generator.add_mark()

    def test_add_swc_connection(self):
        self.vtk_generator.add_swc(self.swc_file_path)
        self.vtk_generator.add_swc_connection(0, 0, 100, 200)

def suite():
    suite = unittest.TestSuite()
    suite.addTests(unittest.makeSuite(TestVtkGenerator))
    return suite

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from contextlib import contextmanager
from mock import patch
import os
import shutil
from tempfile import mkdtemp
from unittest import TestCase

import winbuildtest
from winbuildtest import (
    build_agent,
    create_cloud_agent,
    GO_CMD,
    GOPATH,
)

@contextmanager
def temp_path(module, attr):
    dir_name = mkdtemp()
    old_path = getattr(module, attr)
    if os.path.isabs(old_path):
        rel_path = old_path[1:]
    else:
        rel_path = old_path
    new_path = os.path.join(dir_name, rel_path)
    os.makedirs(new_path)
    try:
        setattr(module, attr, new_path)
        yield new_path
    finally:
        setattr(module, attr, old_path)
        shutil.rmtree(dir_name)

class WinBuildTestTestCase(TestCase):

    def test_build_agent(self):
        with temp_path(winbuildtest, 'JUJUD_CMD_DIR'):
            with patch('winbuildtest.run') as run_mock:
                build_agent()
                args, kwargs = run_mock.call_args
                self.assertEqual((GO_CMD, 'build'), args)
                self.assertEqual('amd64', kwargs['env'].get('GOARCH'))
                self.assertEqual(GOPATH, kwargs['env'].get('GOPATH'))
<commit_msg>Added a test for create_cloud_agent<commit_after>from contextlib import contextmanager
from mock import patch
import os
import shutil
import tarfile
from tempfile import mkdtemp
from unittest import TestCase

import winbuildtest
from winbuildtest import (
    build_agent,
    create_cloud_agent,
    GO_CMD,
    GOPATH,
)

@contextmanager
def temp_path(module, attr):
    dir_name = mkdtemp()
    old_path = getattr(module, attr)
    if os.path.isabs(old_path):
        rel_path = old_path[1:]
    else:
        rel_path = old_path
    new_path = os.path.join(dir_name, rel_path)
    os.makedirs(new_path)
    try:
        setattr(module, attr, new_path)
        yield new_path
    finally:
        setattr(module, attr, old_path)
        shutil.rmtree(dir_name)

class WinBuildTestTestCase(TestCase):

    def test_build_agent(self):
        with temp_path(winbuildtest, 'JUJUD_CMD_DIR'):
            with patch('winbuildtest.run', return_value='built') as run_mock:
                build_agent()
                args, kwargs = run_mock.call_args
                self.assertEqual((GO_CMD, 'build'), args)
                self.assertEqual('amd64', kwargs['env'].get('GOARCH'))
                self.assertEqual(GOPATH, kwargs['env'].get('GOPATH'))

    def test_create_cloud_agent(self):
        with temp_path(winbuildtest, 'CI_DIR') as ci_dir:
            with temp_path(winbuildtest, 'JUJUD_CMD_DIR') as cmd_dir:
                with open('%s/jujud.exe' % cmd_dir, 'w') as fake_jujud:
                    fake_jujud.write('jujud')
                create_cloud_agent('1.20.1')
                agent = '{}/juju-1.20.1-win2012-amd64.tgz'.format(ci_dir)
                self.assertTrue(os.path.isfile(agent))
                with tarfile.open(name=agent, mode='r:gz') as tar:
                    self.assertEqual(['jujud.exe'], tar.getnames())
<|endoftext|>


<commit_before>from typing import Awaitable, Callable, Dict, Iterable, Optional, Tuple, Type, Union

try:
    from typing import Literal, Protocol, TypedDict
except ImportError:
    from typing_extensions import Literal, Protocol, TypedDict  # type: ignore

class ASGIVersions(TypedDict):
    spec_version: str
    version: Union[Literal["2.0"], Literal["3.0"]]

class HTTPScope(TypedDict):
    type: Literal["http"]
    asgi: ASGIVersions
    http_version: str
    method: str
    scheme: str
    path: str
    raw_path: bytes
    query_string: bytes
    root_path: str
    headers: Iterable[Tuple[bytes, bytes]]
    client: Optional[Tuple[str, int]]
    server: Optional[Tuple[str, Optional[int]]]
    extensions: Dict[str, dict]

class WebsocketScope(TypedDict):
    type: Literal["websocket"]
    asgi: ASGIVersions
    http_version: str
    scheme: str
    path: str
    raw_path: bytes
    query_string: bytes
    root_path: str
    headers: Iterable[Tuple[bytes, bytes]]
    client: Optional[Tuple[str, int]]
    server: Optional[Tuple[str, Optional[int]]]
    subprotocols: Iterable[str]
    extensions: Dict[str, dict]

class LifespanScope(TypedDict):
    type: Literal["lifespan"]
    asgi: ASGIVersions

WWWScope = Union[HTTPScope, WebsocketScope]
Scope = Union[HTTPScope, WebsocketScope, LifespanScope]

class HTTPRequestEvent(TypedDict):
    type: Literal["http.request"]
    body: bytes
    more_body: bool

class HTTPResponseStartEvent(TypedDict):
    type: Literal["http.response.start"]
    status: int
    headers: Iterable[Tuple[bytes, bytes]]

class HTTPResponseBodyEvent(TypedDict):
    type: Literal["http.response.body"]
    body: bytes
    more_body: bool

class HTTPServerPushEvent(TypedDict):
    type: Literal["http.response.push"]
    path: str
    headers: Iterable[Tuple[bytes, bytes]]

class HTTPDisconnectEvent(TypedDict):
    type: Literal["http.disconnect"]

class WebsocketConnectEvent(TypedDict):
    type: Literal["websocket.connect"]

class WebsocketAcceptEvent(TypedDict):
    type: Literal["websocket.accept"]
    subprotocol: Optional[str]
    headers: Iterable[Tuple[bytes, bytes]]

class WebsocketReceiveEvent(TypedDict):
    type: Literal["websocket.receive"]
    bytes: Optional[bytes]
    text: Optional[str]

class WebsocketSendEvent(TypedDict):
    type: Literal["websocket.send"]
    bytes: Optional[bytes]
    text: Optional[str]

class WebsocketResponseStartEvent(TypedDict):
    type: Literal["websocket.http.response.start"]
    status: int
    headers: Iterable[Tuple[bytes, bytes]]

class WebsocketResponseBodyEvent(TypedDict):
    type: Literal["websocket.http.response.body"]
    body: bytes
    more_body: bool

class WebsocketDisconnectEvent(TypedDict):
    type: Literal["websocket.disconnect"]
    code: int

class WebsocketCloseEvent(TypedDict):
    type: Literal["websocket.close"]
    code: int

class LifespanStartupEvent(TypedDict):
    type: Literal["lifespan.startup"]

class LifespanShutdownEvent(TypedDict):
    type: Literal["lifespan.shutdown"]

class LifespanStartupCompleteEvent(TypedDict):
    type: Literal["lifespan.startup.complete"]

class LifespanStartupFailedEvent(TypedDict):
    type: Literal["lifespan.startup.failed"]
    message: str

class LifespanShutdownCompleteEvent(TypedDict):
    type: Literal["lifespan.shutdown.complete"]

class LifespanShutdownFailedEvent(TypedDict):
    type: Literal["lifespan.shutdown.failed"]
    message: str

ASGIReceiveEvent = Union[
    HTTPRequestEvent,
    HTTPDisconnectEvent,
    WebsocketConnectEvent,
    WebsocketReceiveEvent,
    WebsocketDisconnectEvent,
    LifespanStartupEvent,
    LifespanShutdownEvent,
]

ASGISendEvent = Union[
    HTTPResponseStartEvent,
    HTTPResponseBodyEvent,
    HTTPServerPushEvent,
    HTTPDisconnectEvent,
    WebsocketAcceptEvent,
    WebsocketSendEvent,
    WebsocketResponseStartEvent,
    WebsocketResponseBodyEvent,
    WebsocketCloseEvent,
    LifespanStartupCompleteEvent,
    LifespanStartupFailedEvent,
    LifespanShutdownCompleteEvent,
    LifespanShutdownFailedEvent,
]

ASGIReceiveCallable = Callable[[], Awaitable[ASGIReceiveEvent]]
ASGISendCallable = Callable[[ASGISendEvent], Awaitable[None]]

class ASGI2Protocol(Protocol):
    def __init__(self, scope: Scope) -> None:
        ...

    async def __call__(
        self, receive: ASGIReceiveCallable, send: ASGISendCallable
    ) -> None:
        ...

ASGI2Application = Type[ASGI2Protocol]
ASGI3Application = Callable[
    [
        Scope,
        ASGIReceiveCallable,
        ASGISendCallable,
    ],
    Awaitable[None],
]
ASGIApplication = Union[ASGI2Application, ASGI3Application]
<commit_msg>Fixed type annotations when run with mypy on Python 3.6<commit_after>import sys
from typing import Awaitable, Callable, Dict, Iterable, Optional, Tuple, Type, Union

if sys.version_info >= (3, 8):
    from typing import Literal, Protocol, TypedDict
else:
    from typing_extensions import Literal, Protocol, TypedDict

class ASGIVersions(TypedDict):
    spec_version: str
    version: Union[Literal["2.0"], Literal["3.0"]]

class HTTPScope(TypedDict):
    type: Literal["http"]
    asgi: ASGIVersions
    http_version: str
    method: str
    scheme: str
    path: str
    raw_path: bytes
    query_string: bytes
    root_path: str
    headers: Iterable[Tuple[bytes, bytes]]
    client: Optional[Tuple[str, int]]
    server: Optional[Tuple[str, Optional[int]]]
    extensions: Dict[str, Dict[object, object]]

class WebsocketScope(TypedDict):
    type: Literal["websocket"]
    asgi: ASGIVersions
    http_version: str
    scheme: str
    path: str
    raw_path: bytes
    query_string: bytes
    root_path: str
    headers: Iterable[Tuple[bytes, bytes]]
    client: Optional[Tuple[str, int]]
    server: Optional[Tuple[str, Optional[int]]]
    subprotocols: Iterable[str]
    extensions: Dict[str, Dict[object, object]]

class LifespanScope(TypedDict):
    type: Literal["lifespan"]
    asgi: ASGIVersions

WWWScope = Union[HTTPScope, WebsocketScope]
Scope = Union[HTTPScope, WebsocketScope, LifespanScope]

class HTTPRequestEvent(TypedDict):
    type: Literal["http.request"]
    body: bytes
    more_body: bool

class HTTPResponseStartEvent(TypedDict):
    type: Literal["http.response.start"]
    status: int
    headers: Iterable[Tuple[bytes, bytes]]

class HTTPResponseBodyEvent(TypedDict):
    type: Literal["http.response.body"]
    body: bytes
    more_body: bool

class HTTPServerPushEvent(TypedDict):
    type: Literal["http.response.push"]
    path: str
    headers: Iterable[Tuple[bytes, bytes]]

class HTTPDisconnectEvent(TypedDict):
    type: Literal["http.disconnect"]

class WebsocketConnectEvent(TypedDict):
    type: Literal["websocket.connect"]

class WebsocketAcceptEvent(TypedDict):
    type: Literal["websocket.accept"]
    subprotocol: Optional[str]
    headers: Iterable[Tuple[bytes, bytes]]

class WebsocketReceiveEvent(TypedDict):
    type: Literal["websocket.receive"]
    bytes: Optional[bytes]
    text: Optional[str]

class WebsocketSendEvent(TypedDict):
    type: Literal["websocket.send"]
    bytes: Optional[bytes]
    text: Optional[str]

class WebsocketResponseStartEvent(TypedDict):
    type: Literal["websocket.http.response.start"]
    status: int
    headers: Iterable[Tuple[bytes, bytes]]

class WebsocketResponseBodyEvent(TypedDict):
    type: Literal["websocket.http.response.body"]
    body: bytes
    more_body: bool

class WebsocketDisconnectEvent(TypedDict):
    type: Literal["websocket.disconnect"]
    code: int

class WebsocketCloseEvent(TypedDict):
    type: Literal["websocket.close"]
    code: int

class LifespanStartupEvent(TypedDict):
    type: Literal["lifespan.startup"]

class LifespanShutdownEvent(TypedDict):
    type: Literal["lifespan.shutdown"]

class LifespanStartupCompleteEvent(TypedDict):
    type: Literal["lifespan.startup.complete"]

class LifespanStartupFailedEvent(TypedDict):
    type: Literal["lifespan.startup.failed"]
    message: str

class LifespanShutdownCompleteEvent(TypedDict):
    type: Literal["lifespan.shutdown.complete"]

class LifespanShutdownFailedEvent(TypedDict):
    type: Literal["lifespan.shutdown.failed"]
    message: str

ASGIReceiveEvent = Union[
    HTTPRequestEvent,
    HTTPDisconnectEvent,
    WebsocketConnectEvent,
    WebsocketReceiveEvent,
    WebsocketDisconnectEvent,
    LifespanStartupEvent,
    LifespanShutdownEvent,
]

ASGISendEvent = Union[
    HTTPResponseStartEvent,
    HTTPResponseBodyEvent,
    HTTPServerPushEvent,
    HTTPDisconnectEvent,
    WebsocketAcceptEvent,
    WebsocketSendEvent,
    WebsocketResponseStartEvent,
    WebsocketResponseBodyEvent,
    WebsocketCloseEvent,
    LifespanStartupCompleteEvent,
    LifespanStartupFailedEvent,
    LifespanShutdownCompleteEvent,
    LifespanShutdownFailedEvent,
]

ASGIReceiveCallable = Callable[[], Awaitable[ASGIReceiveEvent]]
ASGISendCallable = Callable[[ASGISendEvent], Awaitable[None]]

class ASGI2Protocol(Protocol):
    def __init__(self, scope: Scope) -> None:
        ...

    async def __call__(
        self, receive: ASGIReceiveCallable, send: ASGISendCallable
    ) -> None:
        ...

ASGI2Application = Type[ASGI2Protocol]
ASGI3Application = Callable[
    [
        Scope,
        ASGIReceiveCallable,
        ASGISendCallable,
    ],
    Awaitable[None],
]
ASGIApplication = Union[ASGI2Application, ASGI3Application]
<|endoftext|>


<commit_before># Authors: Olivier Grisel <olivier.grisel@ensta.org>
#          Mathieu Blondel <mathieu@mblondel.org>
# License: BSD
import numpy as np
from scipy import sparse
from scipy import linalg
from scipy import stats

from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_greater

from sklearn.utils.extmath import density
from sklearn.utils.extmath import logsumexp
from sklearn.utils.extmath import randomized_svd
from sklearn.utils.extmath import weighted_mode
from sklearn.utils.extmath import cartesian
from sklearn.datasets.samples_generator import make_low_rank_matrix

def test_density():
    rng = np.random.RandomState(0)
    X = rng.randint(10, size=(10, 5))
    X[1, 2] = 0
    X[5, 3] = 0
    X_csr = sparse.csr_matrix(X)
    X_csc = sparse.csc_matrix(X)
    X_coo = sparse.coo_matrix(X)
    X_lil = sparse.lil_matrix(X)

    for X_ in (X_csr, X_csc, X_coo, X_lil):
        assert_equal(density(X_), density(X))

def test_uniform_weights():
    # with uniform weights, results should be identical to stats.mode
    rng = np.random.RandomState(0)
    x = rng.randint(10, size=(10, 5))
    weights = np.ones(x.shape)

    for axis in (None, 0, 1):
        mode, score = stats.mode(x, axis)
        mode2, score2 = weighted_mode(x, weights, axis)

        assert_true(np.all(mode == mode2))
        assert_true(np.all(score == score2))

def test_random_weights():
    # set this up so that each row should have a weighted mode of 6,
    # with a score that is easily reproduced
    mode_result = 6

    rng = np.random.RandomState(0)
    x = rng.randint(mode_result, size=(100, 10))
    w = rng.random_sample(x.shape)

    x[:, :5] = mode_result
    w[:, :5] += 1

    mode, score = weighted_mode(x, w, axis=1)

    assert_true(np.all(mode == mode_result))
    assert_true(np.all(score.ravel() == w[:, :5].sum(1)))

def test_logsumexp():
    # Try to add some smallish numbers in logspace
    x = np.array([1e-40] * 1000000)
    logx = np.log(x)
    assert_almost_equal(np.exp(logsumexp(logx)), x.sum())

    X = np.vstack([x, x])
    logX = np.vstack([logx, logx])
    assert_array_almost_equal(np.exp(logsumexp(logX, axis=0)), X.sum(axis=0))
    assert_array_almost_equal(np.exp(logsumexp(logX, axis=1)), X.sum(axis=1))

def test_randomized_svd_low_rank():
    """Check that extmath.randomized_svd is consistent with linalg.svd"""
    n_samples = 100
    n_features = 500
    rank = 5
    k = 10

    # generate a matrix X of approximate effective rank `rank` and no noise
    # component (very structured signal):
    X = make_low_rank_matrix(n_samples=n_samples, n_features=n_features,
                             effective_rank=rank, tail_strength=0.0,
                             random_state=0)
    assert_equal(X.shape, (n_samples, n_features))

    # compute the singular values of X using the slow exact method
    U, s, V = linalg.svd(X, full_matrices=False)

    # compute the singular values of X using the fast approximate method
    Ua, sa, Va = randomized_svd(X, k)
    assert_equal(Ua.shape, (n_samples, k))
    assert_equal(sa.shape, (k,))
    assert_equal(Va.shape, (k, n_features))

    # ensure that the singular values of both methods are equal up to the real
    # rank of the matrix
    assert_almost_equal(s[:k], sa)

    # check the singular vectors too (while not checking the sign)
    assert_almost_equal(np.dot(U[:, :k], V[:k, :]), np.dot(Ua, Va))

    # check the sparse matrix representation
    X = sparse.csr_matrix(X)

    # compute the singular values of X using the fast approximate method
    Ua, sa, Va = randomized_svd(X, k)
    assert_almost_equal(s[:rank], sa[:rank])

def test_randomized_svd_low_rank_with_noise():
    """Check that extmath.randomized_svd can handle noisy matrices"""
    n_samples = 100
    n_features = 500
    rank = 5
    k = 10

    # generate a matrix X wity structure approximate rank `rank` and an
    # important noisy component
    X = make_low_rank_matrix(n_samples=n_samples, n_features=n_features,
                             effective_rank=rank, tail_strength=0.5,
                             random_state=0)
    assert_equal(X.shape, (n_samples, n_features))

    # compute the singular values of X using the slow exact method
    _, s, _ = linalg.svd(X, full_matrices=False)

    # compute the singular values of X using the fast approximate method
    # without the iterated power method
    _, sa, _ = randomized_svd(X, k, n_iter=0)

    # the approximation does not tolerate the noise:
    assert_greater(np.abs(s[:k] - sa).max(), 0.05)

    # compute the singular values of X using the fast approximate method with
    # iterated power method
    _, sap, _ = randomized_svd(X, k, n_iter=5)

    # the iterated power method is helping getting rid of the noise:
    assert_almost_equal(s[:k], sap, decimal=3)

def test_randomized_svd_infinite_rank():
    """Check that extmath.randomized_svd can handle noisy matrices"""
    n_samples = 100
    n_features = 500
    rank = 5
    k = 10

    # let us try again without 'low_rank component': just regularly but slowly
    # decreasing singular values: the rank of the data matrix is infinite
    X = make_low_rank_matrix(n_samples=n_samples, n_features=n_features,
                             effective_rank=rank, tail_strength=1.0,
                             random_state=0)
    assert_equal(X.shape, (n_samples, n_features))

    # compute the singular values of X using the slow exact method
    _, s, _ = linalg.svd(X, full_matrices=False)

    # compute the singular values of X using the fast approximate method
    # without the iterated power method
    _, sa, _ = randomized_svd(X, k, n_iter=0)

    # the approximation does not tolerate the noise:
    assert_greater(np.abs(s[:k] - sa).max(), 0.1)

    # compute the singular values of X using the fast approximate method with
    # iterated power method
    _, sap, _ = randomized_svd(X, k, n_iter=5)

    # the iterated power method is still managing to get most of the structure
    # at the requested rank
    assert_almost_equal(s[:k], sap, decimal=3)

def test_randomized_svd_transpose_consistency():
    """Check that transposing the design matrix has limit impact"""
    n_samples = 100
    n_features = 500
    rank = 4
    k = 10

    X = make_low_rank_matrix(n_samples=n_samples, n_features=n_features,
                             effective_rank=rank, tail_strength=0.5,
                             random_state=0)
    assert_equal(X.shape, (n_samples, n_features))

    U1, s1, V1 = randomized_svd(X, k, n_iter=3, transpose=False,
                                random_state=0)
    U2, s2, V2 = randomized_svd(X, k, n_iter=3, transpose=True,
                                random_state=0)
    U3, s3, V3 = randomized_svd(X, k, n_iter=3, transpose='auto',
                                random_state=0)
    U4, s4, V4 = linalg.svd(X, full_matrices=False)

    assert_almost_equal(s1, s4[:k], decimal=3)
    assert_almost_equal(s2, s4[:k], decimal=3)
    assert_almost_equal(s3, s4[:k], decimal=3)

    assert_almost_equal(np.dot(U1, V1), np.dot(U4[:, :k], V4[:k, :]),
                        decimal=2)
    assert_almost_equal(np.dot(U2, V2), np.dot(U4[:, :k], V4[:k, :]),
                        decimal=2)

    # in this case 'auto' is equivalent to transpose
    assert_almost_equal(s2, s3)

def test_cartesian():
    """Check if cartesian product delivers the right results"""

    axes = (np.array([1, 2, 3]), np.array([4, 5]), np.array([6, 7]))

    true_out = np.array([[1, 4, 6],
                         [1, 4, 7],
                         [1, 5, 6],
                         [1, 5, 7],
                         [2, 4, 6],
                         [2, 4, 7],
                         [2, 5, 6],
                         [2, 5, 7],
                         [3, 4, 6],
                         [3, 4, 7],
                         [3, 5, 6],
                         [3, 5, 7]])

    out = cartesian(axes)
    assert_array_equal(true_out, out)

    # check single axis
    x = np.arange(3)
    assert_array_equal(x[:, np.newaxis], cartesian((x,)))

def test_randomized_svd_sign_flip():
    a = np.array([[2.0, 0.0], [0.0, 1.0]])
    u1, s1, v1 = randomized_svd(a, 2, flip_sign=True, random_state=41)
    for seed in xrange(100):
        u2, s2, v2 = randomized_svd(a, 2, flip_sign=True, random_state=seed)
        assert_almost_equal(u1, u2)
        assert_almost_equal(v1, v2)
<commit_msg>Make sure svd_flip test actually tests something<commit_after># Authors: Olivier Grisel <olivier.grisel@ensta.org>
#          Mathieu Blondel <mathieu@mblondel.org>
# License: BSD
import numpy as np
from scipy import sparse
from scipy import linalg
from scipy import stats

from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_greater

from sklearn.utils.extmath import density
from sklearn.utils.extmath import logsumexp
from sklearn.utils.extmath import randomized_svd
from sklearn.utils.extmath import svd_flip
from sklearn.utils.extmath import weighted_mode
from sklearn.utils.extmath import cartesian
from sklearn.datasets.samples_generator import make_low_rank_matrix

def test_density():
    rng = np.random.RandomState(0)
    X = rng.randint(10, size=(10, 5))
    X[1, 2] = 0
    X[5, 3] = 0
    X_csr = sparse.csr_matrix(X)
    X_csc = sparse.csc_matrix(X)
    X_coo = sparse.coo_matrix(X)
    X_lil = sparse.lil_matrix(X)

    for X_ in (X_csr, X_csc, X_coo, X_lil):
        assert_equal(density(X_), density(X))

def test_uniform_weights():
    # with uniform weights, results should be identical to stats.mode
    rng = np.random.RandomState(0)
    x = rng.randint(10, size=(10, 5))
    weights = np.ones(x.shape)

    for axis in (None, 0, 1):
        mode, score = stats.mode(x, axis)
        mode2, score2 = weighted_mode(x, weights, axis)

        assert_true(np.all(mode == mode2))
        assert_true(np.all(score == score2))

def test_random_weights():
    # set this up so that each row should have a weighted mode of 6,
    # with a score that is easily reproduced
    mode_result = 6

    rng = np.random.RandomState(0)
    x = rng.randint(mode_result, size=(100, 10))
    w = rng.random_sample(x.shape)

    x[:, :5] = mode_result
    w[:, :5] += 1

    mode, score = weighted_mode(x, w, axis=1)

    assert_true(np.all(mode == mode_result))
    assert_true(np.all(score.ravel() == w[:, :5].sum(1)))

def test_logsumexp():
    # Try to add some smallish numbers in logspace
    x = np.array([1e-40] * 1000000)
    logx = np.log(x)
    assert_almost_equal(np.exp(logsumexp(logx)), x.sum())

    X = np.vstack([x, x])
    logX = np.vstack([logx, logx])
    assert_array_almost_equal(np.exp(logsumexp(logX, axis=0)), X.sum(axis=0))
    assert_array_almost_equal(np.exp(logsumexp(logX, axis=1)), X.sum(axis=1))

def test_randomized_svd_low_rank():
    """Check that extmath.randomized_svd is consistent with linalg.svd"""
    n_samples = 100
    n_features = 500
    rank = 5
    k = 10

    # generate a matrix X of approximate effective rank `rank` and no noise
    # component (very structured signal):
    X = make_low_rank_matrix(n_samples=n_samples, n_features=n_features,
                             effective_rank=rank, tail_strength=0.0,
                             random_state=0)
    assert_equal(X.shape, (n_samples, n_features))

    # compute the singular values of X using the slow exact method
    U, s, V = linalg.svd(X, full_matrices=False)

    # compute the singular values of X using the fast approximate method
    Ua, sa, Va = randomized_svd(X, k)
    assert_equal(Ua.shape, (n_samples, k))
    assert_equal(sa.shape, (k,))
    assert_equal(Va.shape, (k, n_features))

    # ensure that the singular values of both methods are equal up to the real
    # rank of the matrix
    assert_almost_equal(s[:k], sa)

    # check the singular vectors too (while not checking the sign)
    assert_almost_equal(np.dot(U[:, :k], V[:k, :]), np.dot(Ua, Va))

    # check the sparse matrix representation
    X = sparse.csr_matrix(X)

    # compute the singular values of X using the fast approximate method
    Ua, sa, Va = randomized_svd(X, k)
    assert_almost_equal(s[:rank], sa[:rank])

def test_randomized_svd_low_rank_with_noise():
    """Check that extmath.randomized_svd can handle noisy matrices"""
    n_samples = 100
    n_features = 500
    rank = 5
    k = 10

    # generate a matrix X wity structure approximate rank `rank` and an
    # important noisy component
    X = make_low_rank_matrix(n_samples=n_samples, n_features=n_features,
                             effective_rank=rank, tail_strength=0.5,
                             random_state=0)
    assert_equal(X.shape, (n_samples, n_features))

    # compute the singular values of X using the slow exact method
    _, s, _ = linalg.svd(X, full_matrices=False)

    # compute the singular values of X using the fast approximate method
    # without the iterated power method
    _, sa, _ = randomized_svd(X, k, n_iter=0)

    # the approximation does not tolerate the noise:
    assert_greater(np.abs(s[:k] - sa).max(), 0.05)

    # compute the singular values of X using the fast approximate method with
    # iterated power method
    _, sap, _ = randomized_svd(X, k, n_iter=5)

    # the iterated power method is helping getting rid of the noise:
    assert_almost_equal(s[:k], sap, decimal=3)

def test_randomized_svd_infinite_rank():
    """Check that extmath.randomized_svd can handle noisy matrices"""
    n_samples = 100
    n_features = 500
    rank = 5
    k = 10

    # let us try again without 'low_rank component': just regularly but slowly
    # decreasing singular values: the rank of the data matrix is infinite
    X = make_low_rank_matrix(n_samples=n_samples, n_features=n_features,
                             effective_rank=rank, tail_strength=1.0,
                             random_state=0)
    assert_equal(X.shape, (n_samples, n_features))

    # compute the singular values of X using the slow exact method
    _, s, _ = linalg.svd(X, full_matrices=False)

    # compute the singular values of X using the fast approximate method
    # without the iterated power method
    _, sa, _ = randomized_svd(X, k, n_iter=0)

    # the approximation does not tolerate the noise:
    assert_greater(np.abs(s[:k] - sa).max(), 0.1)

    # compute the singular values of X using the fast approximate method with
    # iterated power method
    _, sap, _ = randomized_svd(X, k, n_iter=5)

    # the iterated power method is still managing to get most of the structure
    # at the requested rank
    assert_almost_equal(s[:k], sap, decimal=3)

def test_randomized_svd_transpose_consistency():
    """Check that transposing the design matrix has limit impact"""
    n_samples = 100
    n_features = 500
    rank = 4
    k = 10

    X = make_low_rank_matrix(n_samples=n_samples, n_features=n_features,
                             effective_rank=rank, tail_strength=0.5,
                             random_state=0)
    assert_equal(X.shape, (n_samples, n_features))

    U1, s1, V1 = randomized_svd(X, k, n_iter=3, transpose=False,
                                random_state=0)
    U2, s2, V2 = randomized_svd(X, k, n_iter=3, transpose=True,
                                random_state=0)
    U3, s3, V3 = randomized_svd(X, k, n_iter=3, transpose='auto',
                                random_state=0)
    U4, s4, V4 = linalg.svd(X, full_matrices=False)

    assert_almost_equal(s1, s4[:k], decimal=3)
    assert_almost_equal(s2, s4[:k], decimal=3)
    assert_almost_equal(s3, s4[:k], decimal=3)

    assert_almost_equal(np.dot(U1, V1), np.dot(U4[:, :k], V4[:k, :]),
                        decimal=2)
    assert_almost_equal(np.dot(U2, V2), np.dot(U4[:, :k], V4[:k, :]),
                        decimal=2)

    # in this case 'auto' is equivalent to transpose
    assert_almost_equal(s2, s3)

def test_cartesian():
    """Check if cartesian product delivers the right results"""

    axes = (np.array([1, 2, 3]), np.array([4, 5]), np.array([6, 7]))

    true_out = np.array([[1, 4, 6],
                         [1, 4, 7],
                         [1, 5, 6],
                         [1, 5, 7],
                         [2, 4, 6],
                         [2, 4, 7],
                         [2, 5, 6],
                         [2, 5, 7],
                         [3, 4, 6],
                         [3, 4, 7],
                         [3, 5, 6],
                         [3, 5, 7]])

    out = cartesian(axes)
    assert_array_equal(true_out, out)

    # check single axis
    x = np.arange(3)
    assert_array_equal(x[:, np.newaxis], cartesian((x,)))

def test_randomized_svd_sign_flip():
    a = np.array([[2.0, 0.0], [0.0, 1.0]])
    mismatch = False  # At least one pair should lead to mismatch
    u1, s1, v1 = randomized_svd(a, 2, flip_sign=True, random_state=0)
    for seed in xrange(100):
        u2, s2, v2 = randomized_svd(a, 2, flip_sign=False, random_state=seed)
        if np.any(np.sign(u1) != np.sign(u2)):
            mismatch = True
        u2, s2, v2 = svd_flip(u2, s2, v2)
        assert_almost_equal(u1, u2)
        assert_almost_equal(v1, v2)
    assert_true(mismatch)
<|endoftext|>


<commit_before>import sys

from .client import Client
from .base_manager import BaseManager
from .pubsub_manager import PubSubManager
from .kombu_manager import KombuManager
from .redis_manager import RedisManager
from .zmq_manager import ZmqManager
from .server import Server
from .namespace import Namespace, ClientNamespace
from .middleware import WSGIApp, Middleware
from .tornado import get_tornado_handler
if sys.version_info >= (3, 5):  # pragma: no cover
    from .asyncio_client import AsyncClient
    from .asyncio_server import AsyncServer
    from .asyncio_manager import AsyncManager
    from .asyncio_namespace import AsyncNamespace, AsyncClientNamespace
    from .asyncio_redis_manager import AsyncRedisManager
    from .asgi import ASGIApp
else:  # pragma: no cover
    AsyncClient = None
    AsyncServer = None
    AsyncManager = None
    AsyncNamespace = None
    AsyncRedisManager = None

__version__ = '4.3.0'

__all__ = ['__version__', 'Client', 'Server', 'BaseManager', 'PubSubManager',
           'KombuManager', 'RedisManager', 'ZmqManager', 'Namespace',
           'ClientNamespace', 'WSGIApp', 'Middleware']
if AsyncServer is not None:  # pragma: no cover
    __all__ += ['AsyncClient', 'AsyncServer', 'AsyncNamespace',
                'AsyncClientNamespace', 'AsyncManager', 'AsyncRedisManager',
                'ASGIApp', 'get_tornado_handler']
<commit_msg>Version 4.3.1dev<commit_after>import sys

from .client import Client
from .base_manager import BaseManager
from .pubsub_manager import PubSubManager
from .kombu_manager import KombuManager
from .redis_manager import RedisManager
from .zmq_manager import ZmqManager
from .server import Server
from .namespace import Namespace, ClientNamespace
from .middleware import WSGIApp, Middleware
from .tornado import get_tornado_handler
if sys.version_info >= (3, 5):  # pragma: no cover
    from .asyncio_client import AsyncClient
    from .asyncio_server import AsyncServer
    from .asyncio_manager import AsyncManager
    from .asyncio_namespace import AsyncNamespace, AsyncClientNamespace
    from .asyncio_redis_manager import AsyncRedisManager
    from .asgi import ASGIApp
else:  # pragma: no cover
    AsyncClient = None
    AsyncServer = None
    AsyncManager = None
    AsyncNamespace = None
    AsyncRedisManager = None

__version__ = '4.3.1dev'

__all__ = ['__version__', 'Client', 'Server', 'BaseManager', 'PubSubManager',
           'KombuManager', 'RedisManager', 'ZmqManager', 'Namespace',
           'ClientNamespace', 'WSGIApp', 'Middleware']
if AsyncServer is not None:  # pragma: no cover
    __all__ += ['AsyncClient', 'AsyncServer', 'AsyncNamespace',
                'AsyncClientNamespace', 'AsyncManager', 'AsyncRedisManager',
                'ASGIApp', 'get_tornado_handler']
<|endoftext|>