<commit_before>'''Example

'''

from __future__ import print_function
from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.layers import LSTM
from keras.optimizers import RMSprop
from keras.utils.data_utils import get_file
import numpy as np
import random
import sys
import os
import time

framelen=16
epoch_time = int(time.time())

fname="test/LDC97S44-8k.c2cb"
odir="out/LDC97S44/"+str(epoch_time)+"/"
os.makedirs(odir)
ofname=odir+"out-c2cb-"
modelf=odir+"model-"
#text = open(fname).read()
text = np.fromfile(fname, dtype=np.uint8)
print('corpus length:', len(text))

len_text = len(text)
#len_text = 16000

'''
bytes=[]
for i,t in enumerate(text):
    bytes.append(ord(t))
text = bytes
'''

num_frames = int(len_text / framelen)
#chars = sorted(list(set(text)))
#print('total chars:', len(chars))
#char_indices = dict((c, i) for i, c in enumerate(chars))
#indices_char = dict((i, c) for i, c in enumerate(chars))

# cut the text in semi-redundant sequences of maxlen characters
maxlen = 100
step = 3
sentences = []
next_chars = []
print("scanning byte sequence")

frames = []

for j in range(0, num_frames):
    i = j*framelen
    frames.append(text[i: i+framelen])
print('nb frames:', len(frames), len_text/framelen)
print("scanning frames of length ", len(frames[0]))
for i in range(0, num_frames-maxlen, step):
    # print("getting set from:", i)
    sentences.append(frames[i: i + maxlen])
    next_chars.append(frames[i + maxlen])
print("got sets")
print('nb sequences in bytes:', len(sentences))

print('Vectorization...')
X = np.zeros((len(sentences), maxlen, framelen), dtype=np.uint8)
y = np.zeros((len(sentences), framelen), dtype=np.uint8)
for i, sentence in enumerate(sentences):
    for t, frame in enumerate(sentence):
        #X[i, t, char_indices[char]] = 1
        #for p, prop in enumerate(frame):
         #   X[i, t, p] = (prop)
        X[i,t] = frame
    #y[i, char_indices[next_chars[i]]] = 1
    #for n in range(0, framelen):
    #  y[i, n] = (next_chars[i][n])
    y[i] = next_chars[i]

# build the model: a single LSTM
print('Build model...')
model = Sequential()
model.add(LSTM(128, input_shape=(maxlen, framelen)))
model.add(Dense(framelen))
#model.add(LSTM(128))
model.add(Dense(framelen))
model.add(Dense(framelen))
model.add(Dense(framelen))
#model.add(Activation('softmax'))

optimizer = RMSprop(lr=0.01)
model.compile(loss='mean_absolute_error', optimizer=optimizer)

def sample(preds, temperature=1.0):
    # helper function to sample an index from a probability array
    preds = np.asarray(preds).astype('float64')
    preds = np.round(preds)
   # preds = np.log(preds) / temperature
  #  exp_preds = np.exp(preds)
#    preds = exp_preds / np.sum(exp_preds)
#    probas = np.random.multinomial(1, preds, 1)
    return preds # np.argmax(probas)

# train the model, output generated text after each iteration
for iteration in range(1, 600):
    ofile= open(ofname+str(iteration), "w")
    print()
    print('-' * 50)
    print('Iteration', iteration)
    model.fit(X, y, batch_size=128, nb_epoch=1)

    start_index = random.randint(0, num_frames - maxlen - 1)
    print("start index and maxlen from frames: ", start_index, maxlen, num_frames)

    generated = []
    sentence = frames[start_index: start_index + maxlen]
    generated += ( sentence)
    print('----- Generating with seed: ', str(sentence[0]) )
   # sys.stdout.write(str(generated[0:1]))
  #  sys.stdout.flush()

    for i in range(400):
        x = np.zeros((1, maxlen, framelen))
     #   print("* from range: ", i)
    #    print("sentence length", len(sentence))
        for t, frame in enumerate(sentence):
            #for p in range(0, framelen):
            #    f = frame[p]
               # x[0, t, p] = float(ord(f))
            x[0, t] = frame

        preds = model.predict(x, verbose=0)[0]
        next_frame = sample(preds)

 #           next_char = indices_char[next_index]
     #   intpreds=[]
  #      for p in np.nditer(preds):
   #         intpreds.append(float(round(p)))
   #     next_frame = intpreds
    #    sys.stdout.write(str(next_frame))
    #    sys.stdout.flush()
        generated.append( next_frame)
        sentence = sentence[1:]
        sentence.append( next_frame)
       # print("new sentence length", len(sentence))

    print("last result: ", str(generated[-1]) )
    for i, frame in enumerate(generated):
      #for j,c in enumerate(frame):
        ofile.write((frame))

    ofile.close()
    model.save(modelf+str(iteration))
    print()
<commit_msg>fixed output files<commit_after>'''Example

'''

from __future__ import print_function
from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.layers import LSTM
from keras.optimizers import RMSprop
from keras.utils.data_utils import get_file
import numpy as np
import random
import sys
import os
import time

framelen=16
epoch_time = int(time.time())

fname="test/LDC97S44-8k.c2cb"
odir="out/LDC97S44/"+str(epoch_time)+"/"
os.makedirs(odir)
ofname=odir+"out-c2cb-"
modelf=odir+"model-"
#text = open(fname).read()
text = np.fromfile(fname, dtype=np.uint8)
print('corpus length:', len(text))

len_text = len(text)
#len_text = 16000

'''
bytes=[]
for i,t in enumerate(text):
    bytes.append(ord(t))
text = bytes
'''

num_frames = int(len_text / framelen)
#chars = sorted(list(set(text)))
#print('total chars:', len(chars))
#char_indices = dict((c, i) for i, c in enumerate(chars))
#indices_char = dict((i, c) for i, c in enumerate(chars))

# cut the text in semi-redundant sequences of maxlen characters
maxlen = 100
step = 3
sentences = []
next_chars = []
print("scanning byte sequence")

frames = []

for j in range(0, num_frames):
    i = j*framelen
    frames.append(text[i: i+framelen])
print('nb frames:', len(frames), len_text/framelen)
print("scanning frames of length ", len(frames[0]))
for i in range(0, num_frames-maxlen, step):
    # print("getting set from:", i)
    sentences.append(frames[i: i + maxlen])
    next_chars.append(frames[i + maxlen])
print("got sets")
print('nb sequences in bytes:', len(sentences))

print('Vectorization...')
X = np.zeros((len(sentences), maxlen, framelen), dtype=np.uint8)
y = np.zeros((len(sentences), framelen), dtype=np.uint8)
for i, sentence in enumerate(sentences):
    for t, frame in enumerate(sentence):
        #X[i, t, char_indices[char]] = 1
        #for p, prop in enumerate(frame):
         #   X[i, t, p] = (prop)
        X[i,t] = frame
    #y[i, char_indices[next_chars[i]]] = 1
    #for n in range(0, framelen):
    #  y[i, n] = (next_chars[i][n])
    y[i] = next_chars[i]

# build the model: a single LSTM
print('Build model...')
model = Sequential()
model.add(LSTM(128, input_shape=(maxlen, framelen)))
model.add(Dense(framelen))
#model.add(LSTM(128))
#model.add(Dense(framelen))
model.add(Dense(framelen))
#model.add(Dense(framelen))
#model.add(Activation('softmax'))

optimizer = RMSprop(lr=0.01)
model.compile(loss='mean_absolute_error', optimizer=optimizer)

def sample(preds, temperature=1.0):
    # helper function to sample an index from a probability array
    preds = np.asarray(preds).astype('float64')
    preds = np.round(preds)
   # preds = np.log(preds) / temperature
  #  exp_preds = np.exp(preds)
#    preds = exp_preds / np.sum(exp_preds)
#    probas = np.random.multinomial(1, preds, 1)
    return preds # np.argmax(probas)

# train the model, output generated text after each iteration
for iteration in range(1, 600):
    ofile= open(ofname+str(iteration), "w")
    print()
    print('-' * 50)
    print('Iteration', iteration)
    model.fit(X, y, batch_size=128, nb_epoch=1)

    start_index = random.randint(0, num_frames - maxlen - 1)
    print("start index and maxlen from frames: ", start_index, maxlen, num_frames)

    generated = []
    sentence = frames[start_index: start_index + maxlen]
    generated += ( sentence)
    print('----- Generating with seed: ', str(sentence[0]) )
   # sys.stdout.write(str(generated[0:1]))
  #  sys.stdout.flush()

    for i in range(400):
        x = np.zeros((1, maxlen, framelen))
     #   print("* from range: ", i)
    #    print("sentence length", len(sentence))
        for t, frame in enumerate(sentence):
            #for p in range(0, framelen):
            #    f = frame[p]
               # x[0, t, p] = float(ord(f))
            x[0, t] = frame

        preds = model.predict(x, verbose=0)[0]
        next_frame = sample(preds)

 #           next_char = indices_char[next_index]
     #   intpreds=[]
  #      for p in np.nditer(preds):
   #         intpreds.append(float(round(p)))
   #     next_frame = intpreds
    #    sys.stdout.write(str(next_frame))
    #    sys.stdout.flush()
        generated.append( next_frame)
        sentence = sentence[1:]
        sentence.append( next_frame)
       # print("new sentence length", len(sentence))

#    print("last result: ", str(generated[-1]) )
    for i, frame in enumerate(generated):
      for j,c in enumerate(frame):
        ic = int(c)
        if ic>255:
           ic = 255
        ofile.write(chr(ic))
#        print(str(frame))

      #for j,c in enumerate(frame):
 #       ofile.write((frame))

    ofile.close()
    model.save(modelf+str(iteration))
    print()
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
from __future__ import print_function

__author__ = 'Simon Zheng'

"""Transfer command data from scratch to linux host
"""

import socket

class Server(object):
    def __init__(self, host='', port=50007):
        self.host = host
        self.port = port
        self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.s.bind((self.host, self.port))
        self.conn = None  # received socket connection
        self.address = None  # bind socket address

    def listen(self, maxinum=1, buffer=1024):
        self.s.listen(maxinum)
        self.conn, self.address = self.s.accept()
        print('Connected by', self.address)
        while True:
            data = self.conn.recv(buffer)
            if not data: break
            self.conn.sendall("ok")

    def close(self):
        self.conn.close()

class Client(object):
    def __init__(self, remote_host, port):
        self.host = remote_host
        self.port = port
        self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    def connect(self):
        self.s.connect((self.host, self.port))

    def send(self, msg):
        self.s.sendall(msg)
        data = self.s.recv(1024)
        return data

    def close(self):
        self.s.close()

if __name__ == "__main__":
    host = '192.168.1.219'
    port = 50007

    cli = Client(host,port)
    cli.connect()
    res = cli.send("test msg socket")

    if res == "ok":
        print("test ok")

    cli.close()<commit_msg>add env judge<commit_after># -*- encoding: utf-8 -*-
from __future__ import print_function

__author__ = 'Simon Zheng'

"""Transfer command data from scratch to linux host
"""

import socket

class Server(object):
    def __init__(self, host='', port=50007):
        self.host = host
        self.port = port
        self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.s.bind((self.host, self.port))
        self.conn = None  # received socket connection
        self.address = None  # bind socket address

    def listen(self, maxinum=1, buffer=1024):
        self.s.listen(maxinum)
        self.conn, self.address = self.s.accept()
        print('Connected by', self.address)
        while True:
            data = self.conn.recv(buffer)
            if not data: break
            self.conn.sendall("ok")

    def close(self):
        self.conn.close()

class Client(object):
    def __init__(self, remote_host, port):
        self.host = remote_host
        self.port = port
        self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    def connect(self):
        self.s.connect((self.host, self.port))

    def send(self, msg):
        self.s.sendall(msg)
        data = self.s.recv(1024)
        return data

    def close(self):
        self.s.close()

if __name__ == "__main__":
    import os

    if os.getenv('ENV', None) == 'client' or os.getenv('ENV', None) == None:
        host = '192.168.1.219'
        port = 50007

        cli = Client(host, port)
        cli.connect()
        res = cli.send("Test msg socket")
        if res == "ok":
            print("test ok")

        cli.close()
    elif os.getenv('ENV', None) == 'server':
        ser = Server()
        ser.listen()
        ser.close()
    else:
        print("Please set your EVN var of system")
<|endoftext|>


<commit_before># Copyright 2021 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import time
from typing import Tuple

from absl import flags
from absl.testing import absltest

from framework import xds_k8s_flags
from framework import xds_url_map_testcase
from framework.infrastructure import traffic_director
from framework.rpc import grpc_channelz
from framework.test_app import client_app

# Type aliases
HostRule = xds_url_map_testcase.HostRule
PathMatcher = xds_url_map_testcase.PathMatcher
GcpResourceManager = xds_url_map_testcase.GcpResourceManager
DumpedXdsConfig = xds_url_map_testcase.DumpedXdsConfig
RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall
RpcTypeEmptyCall = xds_url_map_testcase.RpcTypeEmptyCall
XdsTestClient = client_app.XdsTestClient

logger = logging.getLogger(__name__)
flags.adopt_module_key_flags(xds_url_map_testcase)

_NUM_RPCS = 150
_TEST_METADATA_KEY = traffic_director.TEST_AFFINITY_METADATA_KEY
_TEST_METADATA_VALUE_UNARY = 'unary_yranu'
_TEST_METADATA_VALUE_EMPTY = 'empty_ytpme'
_TEST_METADATA_NUMERIC_KEY = 'xds_md_numeric'
_TEST_METADATA_NUMERIC_VALUE = '159'

_TEST_METADATA = (
    (RpcTypeUnaryCall, _TEST_METADATA_KEY, _TEST_METADATA_VALUE_UNARY),
    (RpcTypeEmptyCall, _TEST_METADATA_KEY, _TEST_METADATA_VALUE_EMPTY),
    (RpcTypeUnaryCall, _TEST_METADATA_NUMERIC_KEY,
     _TEST_METADATA_NUMERIC_VALUE),
)

_ChannelzChannelState = grpc_channelz.ChannelState

class TestHeaderBasedAffinity(xds_url_map_testcase.XdsUrlMapTestCase):

    @staticmethod
    def supported_clients() -> Tuple[str]:
        return 'cpp', 'java'

    @staticmethod
    def client_init_config(rpc: str, metadata: str):
        # Config the init RPCs to send with the same set of metadata. Without
        # this, the init RPCs will not have headers, and will pick random
        # backends (behavior of RING_HASH). This is necessary to only one
        # sub-channel is picked and used from the beginning, thus the channel
        # will only create this one sub-channel.
        return 'EmptyCall', 'EmptyCall:%s:%s' % (_TEST_METADATA_KEY,
                                                 _TEST_METADATA_VALUE_EMPTY)

    @staticmethod
    def url_map_change(
            host_rule: HostRule,
            path_matcher: PathMatcher) -> Tuple[HostRule, PathMatcher]:
        # Update default service to the affinity service.
        path_matcher["defaultService"] = GcpResourceManager(
        ).affinity_backend_service()
        return host_rule, path_matcher

    def xds_config_validate(self, xds_config: DumpedXdsConfig):
        # 3 endpoints in the affinity backend service.
        self.assertNumEndpoints(xds_config, 3)
        self.assertEqual(
            xds_config.rds['virtualHosts'][0]['routes'][0]['route']
            ['hashPolicy'][0]['header']['headerName'], _TEST_METADATA_KEY)
        self.assertEqual(xds_config.cds[0]['lbPolicy'], 'RING_HASH')

    def rpc_distribution_validate(self, test_client: XdsTestClient):
        rpc_distribution = self.configure_and_send(test_client,
                                                   rpc_types=[RpcTypeEmptyCall],
                                                   metadata=_TEST_METADATA,
                                                   num_rpcs=_NUM_RPCS)
        # Only one backend should receive traffic, even though there are 3
        # backends.
        self.assertEqual(1, rpc_distribution.num_peers)
        self.assertLen(
            test_client.find_subchannels_with_state(
                _ChannelzChannelState.READY),
            1,
        )
        self.assertLen(
            test_client.find_subchannels_with_state(_ChannelzChannelState.IDLE),
            2,
        )
        # Send 150 RPCs without headers. RPCs without headers will pick random
        # backends. After this, we expect to see all backends to be connected.
        rpc_distribution = self.configure_and_send(
            test_client,
            rpc_types=[RpcTypeEmptyCall, RpcTypeUnaryCall],
            num_rpcs=_NUM_RPCS)
        self.assertEqual(3, rpc_distribution.num_peers)
        self.assertLen(
            test_client.find_subchannels_with_state(
                _ChannelzChannelState.READY),
            3,
        )

class TestHeaderBasedAffinityMultipleHeaders(
        xds_url_map_testcase.XdsUrlMapTestCase):

    @staticmethod
    def supported_clients() -> Tuple[str]:
        return 'cpp', 'java'

    @staticmethod
    def client_init_config(rpc: str, metadata: str):
        # Config the init RPCs to send with the same set of metadata. Without
        # this, the init RPCs will not have headers, and will pick random
        # backends (behavior of RING_HASH). This is necessary to only one
        # sub-channel is picked and used from the beginning, thus the channel
        # will only create this one sub-channel.
        return 'EmptyCall', 'EmptyCall:%s:%s' % (_TEST_METADATA_KEY,
                                                 _TEST_METADATA_VALUE_EMPTY)

    @staticmethod
    def url_map_change(
            host_rule: HostRule,
            path_matcher: PathMatcher) -> Tuple[HostRule, PathMatcher]:
        # Update default service to the affinity service.
        path_matcher["defaultService"] = GcpResourceManager(
        ).affinity_backend_service()
        return host_rule, path_matcher

    def xds_config_validate(self, xds_config: DumpedXdsConfig):
        # 3 endpoints in the affinity backend service.
        self.assertNumEndpoints(xds_config, 3)
        self.assertEqual(
            xds_config.rds['virtualHosts'][0]['routes'][0]['route']
            ['hashPolicy'][0]['header']['headerName'], _TEST_METADATA_KEY)
        self.assertEqual(xds_config.cds[0]['lbPolicy'], 'RING_HASH')

    def rpc_distribution_validate(self, test_client: XdsTestClient):
        rpc_distribution = self.configure_and_send(test_client,
                                                   rpc_types=[RpcTypeEmptyCall],
                                                   metadata=_TEST_METADATA,
                                                   num_rpcs=_NUM_RPCS)
        # Only one backend should receive traffic, even though there are 3
        # backends.
        self.assertEqual(1, rpc_distribution.num_peers)
        self.assertLen(
            test_client.find_subchannels_with_state(
                _ChannelzChannelState.READY),
            1,
        )
        self.assertLen(
            test_client.find_subchannels_with_state(_ChannelzChannelState.IDLE),
            2,
        )
        empty_call_peer = list(rpc_distribution.raw['rpcsByMethod']['EmptyCall']
                               ['rpcsByPeer'].keys())[0]
        # Send RPCs with a different metadata value, try different values to
        # verify that the client will pick a different backend.
        #
        # EmptyCalls will be sent with the same metadata as before, and
        # UnaryCalls will be sent with headers from ["0".."29"]. We check the
        # endpoint picked for UnaryCall, and stop as soon as one different from
        # the EmptyCall peer is picked.
        #
        # Note that there's a small chance all the headers would still pick the
        # same backend used by EmptyCall. But there will be over a thousand
        # nodes on the ring (default min size is 1024), and the probability of
        # picking the same backend should be fairly small.
        different_peer_picked = False
        for i in range(30):
            new_metadata = (
                (RpcTypeEmptyCall, _TEST_METADATA_KEY,
                 _TEST_METADATA_VALUE_EMPTY),
                (RpcTypeUnaryCall, _TEST_METADATA_KEY, str(i)),
            )
            rpc_distribution = self.configure_and_send(
                test_client,
                rpc_types=[RpcTypeEmptyCall, RpcTypeUnaryCall],
                metadata=new_metadata,
                num_rpcs=_NUM_RPCS)
            unary_call_peer = list(rpc_distribution.raw['rpcsByMethod']
                                   ['UnaryCall']['rpcsByPeer'].keys())[0]
            if unary_call_peer != empty_call_peer:
                different_peer_picked = True
                break
        self.assertTrue(
            different_peer_picked,
            "the same endpoint was picked for all the headers, expect a different endpoint to be picked"
        )
        self.assertLen(
            test_client.find_subchannels_with_state(
                _ChannelzChannelState.READY),
            2,
        )
        self.assertLen(
            test_client.find_subchannels_with_state(_ChannelzChannelState.IDLE),
            1,
        )

# TODO: add more test cases
# 1. based on the basic test, turn down the backend in use, then verify that all
#    RPCs are sent to another backend

if __name__ == '__main__':
    absltest.main()
<commit_msg>xds/interop: turn on affinity urlmap tests for go (#27359)<commit_after># Copyright 2021 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import time
from typing import Tuple

from absl import flags
from absl.testing import absltest

from framework import xds_k8s_flags
from framework import xds_url_map_testcase
from framework.infrastructure import traffic_director
from framework.rpc import grpc_channelz
from framework.test_app import client_app

# Type aliases
HostRule = xds_url_map_testcase.HostRule
PathMatcher = xds_url_map_testcase.PathMatcher
GcpResourceManager = xds_url_map_testcase.GcpResourceManager
DumpedXdsConfig = xds_url_map_testcase.DumpedXdsConfig
RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall
RpcTypeEmptyCall = xds_url_map_testcase.RpcTypeEmptyCall
XdsTestClient = client_app.XdsTestClient

logger = logging.getLogger(__name__)
flags.adopt_module_key_flags(xds_url_map_testcase)

_NUM_RPCS = 150
_TEST_METADATA_KEY = traffic_director.TEST_AFFINITY_METADATA_KEY
_TEST_METADATA_VALUE_UNARY = 'unary_yranu'
_TEST_METADATA_VALUE_EMPTY = 'empty_ytpme'
_TEST_METADATA_NUMERIC_KEY = 'xds_md_numeric'
_TEST_METADATA_NUMERIC_VALUE = '159'

_TEST_METADATA = (
    (RpcTypeUnaryCall, _TEST_METADATA_KEY, _TEST_METADATA_VALUE_UNARY),
    (RpcTypeEmptyCall, _TEST_METADATA_KEY, _TEST_METADATA_VALUE_EMPTY),
    (RpcTypeUnaryCall, _TEST_METADATA_NUMERIC_KEY,
     _TEST_METADATA_NUMERIC_VALUE),
)

_ChannelzChannelState = grpc_channelz.ChannelState

class TestHeaderBasedAffinity(xds_url_map_testcase.XdsUrlMapTestCase):

    @staticmethod
    def supported_clients() -> Tuple[str]:
        return 'cpp', 'java', 'go'

    @staticmethod
    def client_init_config(rpc: str, metadata: str):
        # Config the init RPCs to send with the same set of metadata. Without
        # this, the init RPCs will not have headers, and will pick random
        # backends (behavior of RING_HASH). This is necessary to only one
        # sub-channel is picked and used from the beginning, thus the channel
        # will only create this one sub-channel.
        return 'EmptyCall', 'EmptyCall:%s:%s' % (_TEST_METADATA_KEY,
                                                 _TEST_METADATA_VALUE_EMPTY)

    @staticmethod
    def url_map_change(
            host_rule: HostRule,
            path_matcher: PathMatcher) -> Tuple[HostRule, PathMatcher]:
        # Update default service to the affinity service.
        path_matcher["defaultService"] = GcpResourceManager(
        ).affinity_backend_service()
        return host_rule, path_matcher

    def xds_config_validate(self, xds_config: DumpedXdsConfig):
        # 3 endpoints in the affinity backend service.
        self.assertNumEndpoints(xds_config, 3)
        self.assertEqual(
            xds_config.rds['virtualHosts'][0]['routes'][0]['route']
            ['hashPolicy'][0]['header']['headerName'], _TEST_METADATA_KEY)
        self.assertEqual(xds_config.cds[0]['lbPolicy'], 'RING_HASH')

    def rpc_distribution_validate(self, test_client: XdsTestClient):
        rpc_distribution = self.configure_and_send(test_client,
                                                   rpc_types=[RpcTypeEmptyCall],
                                                   metadata=_TEST_METADATA,
                                                   num_rpcs=_NUM_RPCS)
        # Only one backend should receive traffic, even though there are 3
        # backends.
        self.assertEqual(1, rpc_distribution.num_peers)
        self.assertLen(
            test_client.find_subchannels_with_state(
                _ChannelzChannelState.READY),
            1,
        )
        self.assertLen(
            test_client.find_subchannels_with_state(_ChannelzChannelState.IDLE),
            2,
        )
        # Send 150 RPCs without headers. RPCs without headers will pick random
        # backends. After this, we expect to see all backends to be connected.
        rpc_distribution = self.configure_and_send(
            test_client,
            rpc_types=[RpcTypeEmptyCall, RpcTypeUnaryCall],
            num_rpcs=_NUM_RPCS)
        self.assertEqual(3, rpc_distribution.num_peers)
        self.assertLen(
            test_client.find_subchannels_with_state(
                _ChannelzChannelState.READY),
            3,
        )

class TestHeaderBasedAffinityMultipleHeaders(
        xds_url_map_testcase.XdsUrlMapTestCase):

    @staticmethod
    def supported_clients() -> Tuple[str]:
        return 'cpp', 'java', 'go'

    @staticmethod
    def client_init_config(rpc: str, metadata: str):
        # Config the init RPCs to send with the same set of metadata. Without
        # this, the init RPCs will not have headers, and will pick random
        # backends (behavior of RING_HASH). This is necessary to only one
        # sub-channel is picked and used from the beginning, thus the channel
        # will only create this one sub-channel.
        return 'EmptyCall', 'EmptyCall:%s:%s' % (_TEST_METADATA_KEY,
                                                 _TEST_METADATA_VALUE_EMPTY)

    @staticmethod
    def url_map_change(
            host_rule: HostRule,
            path_matcher: PathMatcher) -> Tuple[HostRule, PathMatcher]:
        # Update default service to the affinity service.
        path_matcher["defaultService"] = GcpResourceManager(
        ).affinity_backend_service()
        return host_rule, path_matcher

    def xds_config_validate(self, xds_config: DumpedXdsConfig):
        # 3 endpoints in the affinity backend service.
        self.assertNumEndpoints(xds_config, 3)
        self.assertEqual(
            xds_config.rds['virtualHosts'][0]['routes'][0]['route']
            ['hashPolicy'][0]['header']['headerName'], _TEST_METADATA_KEY)
        self.assertEqual(xds_config.cds[0]['lbPolicy'], 'RING_HASH')

    def rpc_distribution_validate(self, test_client: XdsTestClient):
        rpc_distribution = self.configure_and_send(test_client,
                                                   rpc_types=[RpcTypeEmptyCall],
                                                   metadata=_TEST_METADATA,
                                                   num_rpcs=_NUM_RPCS)
        # Only one backend should receive traffic, even though there are 3
        # backends.
        self.assertEqual(1, rpc_distribution.num_peers)
        self.assertLen(
            test_client.find_subchannels_with_state(
                _ChannelzChannelState.READY),
            1,
        )
        self.assertLen(
            test_client.find_subchannels_with_state(_ChannelzChannelState.IDLE),
            2,
        )
        empty_call_peer = list(rpc_distribution.raw['rpcsByMethod']['EmptyCall']
                               ['rpcsByPeer'].keys())[0]
        # Send RPCs with a different metadata value, try different values to
        # verify that the client will pick a different backend.
        #
        # EmptyCalls will be sent with the same metadata as before, and
        # UnaryCalls will be sent with headers from ["0".."29"]. We check the
        # endpoint picked for UnaryCall, and stop as soon as one different from
        # the EmptyCall peer is picked.
        #
        # Note that there's a small chance all the headers would still pick the
        # same backend used by EmptyCall. But there will be over a thousand
        # nodes on the ring (default min size is 1024), and the probability of
        # picking the same backend should be fairly small.
        different_peer_picked = False
        for i in range(30):
            new_metadata = (
                (RpcTypeEmptyCall, _TEST_METADATA_KEY,
                 _TEST_METADATA_VALUE_EMPTY),
                (RpcTypeUnaryCall, _TEST_METADATA_KEY, str(i)),
            )
            rpc_distribution = self.configure_and_send(
                test_client,
                rpc_types=[RpcTypeEmptyCall, RpcTypeUnaryCall],
                metadata=new_metadata,
                num_rpcs=_NUM_RPCS)
            unary_call_peer = list(rpc_distribution.raw['rpcsByMethod']
                                   ['UnaryCall']['rpcsByPeer'].keys())[0]
            if unary_call_peer != empty_call_peer:
                different_peer_picked = True
                break
        self.assertTrue(
            different_peer_picked,
            "the same endpoint was picked for all the headers, expect a different endpoint to be picked"
        )
        self.assertLen(
            test_client.find_subchannels_with_state(
                _ChannelzChannelState.READY),
            2,
        )
        self.assertLen(
            test_client.find_subchannels_with_state(_ChannelzChannelState.IDLE),
            1,
        )

# TODO: add more test cases
# 1. based on the basic test, turn down the backend in use, then verify that all
#    RPCs are sent to another backend

if __name__ == '__main__':
    absltest.main()
<|endoftext|>


<commit_before>"""
Test compiled module
"""
import os
import platform
import sys
import jedi

from .helpers import cwd_at

@cwd_at('test/extensions')
def test_completions():
    if platform.architecture()[0] == '64bit':
        package_name = "compiled%s%s" % sys.version_info[:2]
        sys.path.insert(0, os.getcwd())
        if os.path.exists(package_name):
            s = jedi.Script("from %s import compiled; compiled." % package_name)
            assert len(s.completions()) >= 2

@cwd_at('test/extensions')
def test_call_signatures_extension():
    # with a cython extension
    if platform.architecture()[0] == '64bit':
        package_name = "compiled%s%s" % sys.version_info[:2]
        sys.path.insert(0, os.getcwd())
        if os.path.exists(package_name):
            s = jedi.Script("from %s import compiled; compiled.Foo(" %
                            package_name)
            defs = s.call_signatures()
            for call_def in defs:
                for param in call_def.params:
                    pass

def test_call_signatures_stdlib():
    code = "import math; math.cos("
    s = jedi.Script(code)
    defs = s.call_signatures()
    for call_def in defs:
        for p in call_def.params:
            assert str(p) == 'x'
<commit_msg>Fix test (bad assert)<commit_after>"""
Test compiled module
"""
import os
import platform
import sys
import jedi

from .helpers import cwd_at

@cwd_at('test/extensions')
def test_completions():
    if platform.architecture()[0] == '64bit':
        package_name = "compiled%s%s" % sys.version_info[:2]
        sys.path.insert(0, os.getcwd())
        if os.path.exists(package_name):
            s = jedi.Script("from %s import compiled; compiled." % package_name)
            assert len(s.completions()) >= 2

@cwd_at('test/extensions')
def test_call_signatures_extension():
    # with a cython extension
    if platform.architecture()[0] == '64bit':
        package_name = "compiled%s%s" % sys.version_info[:2]
        sys.path.insert(0, os.getcwd())
        if os.path.exists(package_name):
            s = jedi.Script("from %s import compiled; compiled.Foo(" %
                            package_name)
            defs = s.call_signatures()
            for call_def in defs:
                for param in call_def.params:
                    pass

def test_call_signatures_stdlib():
    code = "import math; math.cos("
    s = jedi.Script(code)
    defs = s.call_signatures()
    for call_def in defs:
        assert len(call_def.params) == 1
<|endoftext|>


<commit_before>#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2014 Mozilla Corporation

import argparse

from datetime import datetime, timedelta
from time import sleep
from configlib import getConfig
import json
import time
import os
import sys

from elasticsearch.exceptions import ConnectionError
import requests

from mozdef_util.elasticsearch_client import ElasticsearchClient

parser = argparse.ArgumentParser(description='Create the correct indexes and aliases in elasticsearch')
parser.add_argument('esserver', help='Elasticsearch server (ex: http://elasticsearch:9200)')
parser.add_argument('default_mapping_file', help='The relative path to default mapping json file (ex: cron/defaultMappingTemplate.json)')
parser.add_argument('state_mapping_file', help='The relative path to state mapping json file (ex: cron/mozdefStateDefaultMappingTemplate.json)')
parser.add_argument('backup_conf_file', help='The relative path to backup.conf file (ex: cron/backup.conf)')
parser.add_argument('kibana_url', help='The URL of the kibana endpoint (ex: http://kibana:5601)')
args = parser.parse_args()

esserver = os.environ.get('OPTIONS_ESSERVERS')
if esserver is None:
    esserver = args.esserver
esserver = esserver.strip('/')
print "Connecting to " + esserver
client = ElasticsearchClient(esserver)

kibana_url = os.environ.get('OPTIONS_KIBANAURL', args.kibana_url)

current_date = datetime.now()
event_index_name = current_date.strftime("events-%Y%m%d")
previous_event_index_name = (current_date - timedelta(days=1)).strftime("events-%Y%m%d")
weekly_index_alias = 'events-weekly'
alert_index_name = current_date.strftime("alerts-%Y%m")

kibana_index_name = '.kibana_1'
# For this version of kibana, they require specifying
# the kibana version via the api for setting
# the default index, seems weird, but it is what it is.
kibana_version = '6.6.2'

state_index_name = 'mozdefstate'

index_settings_str = ''
with open(args.default_mapping_file) as data_file:
    index_settings_str = data_file.read()

index_settings = json.loads(index_settings_str)

state_index_settings_str = ''
with open(args.state_mapping_file) as data_file:
    state_index_settings_str = data_file.read()

state_index_settings = json.loads(state_index_settings_str)

all_indices = []
total_num_tries = 15
for attempt in range(total_num_tries):
    try:
        all_indices = client.get_indices()
    except ConnectionError:
        print 'Unable to connect to Elasticsearch...retrying'
        sleep(5)
    else:
        break
else:
    print 'Cannot connect to Elasticsearch after ' + str(total_num_tries) + ' tries, exiting script.'
    exit(1)

refresh_interval = getConfig('refresh_interval', '1s', args.backup_conf_file)
number_of_shards = getConfig('number_of_shards', '1', args.backup_conf_file)
number_of_replicas = getConfig('number_of_replicas', '1', args.backup_conf_file)
slowlog_threshold_query_warn = getConfig('slowlog_threshold_query_warn', '5s', args.backup_conf_file)
slowlog_threshold_fetch_warn = getConfig('slowlog_threshold_fetch_warn', '5s', args.backup_conf_file)
mapping_total_fields_limit = getConfig('mapping_total_fields_limit', '1000', args.backup_conf_file)

index_options = {
    "index": {
        "refresh_interval": refresh_interval,
        "number_of_shards": number_of_shards,
        "number_of_replicas": number_of_replicas,
        "search.slowlog.threshold.query.warn": slowlog_threshold_query_warn,
        "search.slowlog.threshold.fetch.warn": slowlog_threshold_fetch_warn,
        "mapping.total_fields.limit": mapping_total_fields_limit
    }
}
index_settings['settings'] = index_options
state_index_settings['settings'] = index_options

# Create initial indices
if event_index_name not in all_indices:
    print "Creating " + event_index_name
    client.create_index(event_index_name, index_config=index_settings)
client.create_alias('events', event_index_name)

if previous_event_index_name not in all_indices:
    print "Creating " + previous_event_index_name
    client.create_index(previous_event_index_name, index_config=index_settings)
client.create_alias('events-previous', previous_event_index_name)

if alert_index_name not in all_indices:
    print "Creating " + alert_index_name
    client.create_index(alert_index_name, index_config=index_settings)
client.create_alias('alerts', alert_index_name)

if weekly_index_alias not in all_indices:
    print "Creating " + weekly_index_alias
    client.create_alias_multiple_indices(weekly_index_alias, [event_index_name, previous_event_index_name])

if state_index_name not in all_indices:
    print "Creating " + state_index_name
    client.create_index(state_index_name, index_config=state_index_settings)

# Wait for kibana service to get ready
num_times = 0
while not requests.get(kibana_url).ok:
    if num_times < 5:
        print("Waiting for kibana index to be ready")
        time.sleep(2)
        num_times += 1
    else:
        print("Kibana service never started up...exiting")
        sys.exit(1)

# Check if index-patterns already exist
if kibana_index_name in client.get_indices():
    existing_patterns_url = kibana_url + "/api/saved_objects/_find?type=index-pattern&search_fields=title&search=*"
    resp = requests.get(url=existing_patterns_url)
    existing_patterns = json.loads(resp.text)
    if len(existing_patterns['saved_objects']) > 0:
        print("Index patterns already exist, exiting script early")
        sys.exit(0)

# Create index-patterns
kibana_headers = {'content-type': 'application/json', 'kbn-xsrf': 'true'}
index_mappings_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'index_mappings')
listing = os.listdir(index_mappings_path)
for infile in listing:
    json_file_path = os.path.join(index_mappings_path, infile)
    with open(json_file_path) as json_data:
        mapping_data = json.load(json_data)
        index_name = mapping_data['attributes']['title']
        print "Creating {0} index mapping".format(index_name)
        mapping_url = kibana_url + "/api/saved_objects/index-pattern/" + index_name
        resp = requests.post(url=mapping_url, data=json.dumps(mapping_data), headers=kibana_headers)
        if not resp.ok:
            print("Unable to create index mapping: " + resp.text)

# Remove existing default index mapping if it exists
resp = requests.delete(url=kibana_url + "/api/saved_objects/config/" + kibana_version, headers=kibana_headers)
if not resp.ok:
    print("Unable to delete existing default index mapping: {} {}".format(resp.status_code, resp.content))

# Set default index mapping to events-*
data = {
    "attributes": {
        "buildNum": "19548",
        "defaultIndex": "events-*"
    }
}
print("Creating default index pattern for events-*")
resp = requests.post(url=kibana_url + "/api/saved_objects/config/" + kibana_version, data=json.dumps(data), headers=kibana_headers)
if not resp.ok:
    print("Failed to set default index: {} {}".format(resp.status_code, resp.content))

# Check if dashboards already exist
if kibana_index_name in client.get_indices():
    existing_patterns_url = kibana_url + "/api/saved_objects/_find?type=dashboard&search_fields=title&search=*"
    resp = requests.get(url=existing_patterns_url)
    existing_patterns = json.loads(resp.text)
    if len(existing_patterns['saved_objects']) > 0:
        print("Dashboards already exist, exiting script early")
        sys.exit(0)

# Create visualizations/dashboards
dashboards_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'resources')
listing = os.listdir(dashboards_path)
for infile in listing:
    json_file_path = os.path.join(dashboards_path, infile)
    with open(json_file_path) as json_data:
        mapping_data = json.load(json_data)
        mapping_type = mapping_data['type']
        print("Creating {0} {1}".format(
            mapping_data[mapping_type]['title'],
            mapping_type
        ))
        post_data = {
            "attributes": mapping_data[mapping_type]
        }
        # We use the filename as the id of the resource
        resource_name = infile.replace('.json', '')
        kibana_type_url = kibana_url + "/api/saved_objects/" + mapping_type + "/" + resource_name
        requests.post(url=kibana_type_url, data=json.dumps(post_data), headers=kibana_headers)
<commit_msg>Update wait for kibana in initial setup script<commit_after>#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2014 Mozilla Corporation

import argparse

from datetime import datetime, timedelta
from time import sleep
from configlib import getConfig
import json
import time
import os
import sys

from elasticsearch.exceptions import ConnectionError
import requests

from mozdef_util.elasticsearch_client import ElasticsearchClient

parser = argparse.ArgumentParser(description='Create the correct indexes and aliases in elasticsearch')
parser.add_argument('esserver', help='Elasticsearch server (ex: http://elasticsearch:9200)')
parser.add_argument('default_mapping_file', help='The relative path to default mapping json file (ex: cron/defaultMappingTemplate.json)')
parser.add_argument('state_mapping_file', help='The relative path to state mapping json file (ex: cron/mozdefStateDefaultMappingTemplate.json)')
parser.add_argument('backup_conf_file', help='The relative path to backup.conf file (ex: cron/backup.conf)')
parser.add_argument('kibana_url', help='The URL of the kibana endpoint (ex: http://kibana:5601)')
args = parser.parse_args()

esserver = os.environ.get('OPTIONS_ESSERVERS')
if esserver is None:
    esserver = args.esserver
esserver = esserver.strip('/')
print "Connecting to " + esserver
client = ElasticsearchClient(esserver)

kibana_url = os.environ.get('OPTIONS_KIBANAURL', args.kibana_url)

current_date = datetime.now()
event_index_name = current_date.strftime("events-%Y%m%d")
previous_event_index_name = (current_date - timedelta(days=1)).strftime("events-%Y%m%d")
weekly_index_alias = 'events-weekly'
alert_index_name = current_date.strftime("alerts-%Y%m")

kibana_index_name = '.kibana_1'
# For this version of kibana, they require specifying
# the kibana version via the api for setting
# the default index, seems weird, but it is what it is.
kibana_version = '6.6.2'

state_index_name = 'mozdefstate'

index_settings_str = ''
with open(args.default_mapping_file) as data_file:
    index_settings_str = data_file.read()

index_settings = json.loads(index_settings_str)

state_index_settings_str = ''
with open(args.state_mapping_file) as data_file:
    state_index_settings_str = data_file.read()

state_index_settings = json.loads(state_index_settings_str)

all_indices = []
total_num_tries = 15
for attempt in range(total_num_tries):
    try:
        all_indices = client.get_indices()
    except ConnectionError:
        print 'Unable to connect to Elasticsearch...retrying'
        sleep(5)
    else:
        break
else:
    print 'Cannot connect to Elasticsearch after ' + str(total_num_tries) + ' tries, exiting script.'
    exit(1)

refresh_interval = getConfig('refresh_interval', '1s', args.backup_conf_file)
number_of_shards = getConfig('number_of_shards', '1', args.backup_conf_file)
number_of_replicas = getConfig('number_of_replicas', '1', args.backup_conf_file)
slowlog_threshold_query_warn = getConfig('slowlog_threshold_query_warn', '5s', args.backup_conf_file)
slowlog_threshold_fetch_warn = getConfig('slowlog_threshold_fetch_warn', '5s', args.backup_conf_file)
mapping_total_fields_limit = getConfig('mapping_total_fields_limit', '1000', args.backup_conf_file)

index_options = {
    "index": {
        "refresh_interval": refresh_interval,
        "number_of_shards": number_of_shards,
        "number_of_replicas": number_of_replicas,
        "search.slowlog.threshold.query.warn": slowlog_threshold_query_warn,
        "search.slowlog.threshold.fetch.warn": slowlog_threshold_fetch_warn,
        "mapping.total_fields.limit": mapping_total_fields_limit
    }
}
index_settings['settings'] = index_options
state_index_settings['settings'] = index_options

# Create initial indices
if event_index_name not in all_indices:
    print "Creating " + event_index_name
    client.create_index(event_index_name, index_config=index_settings)
client.create_alias('events', event_index_name)

if previous_event_index_name not in all_indices:
    print "Creating " + previous_event_index_name
    client.create_index(previous_event_index_name, index_config=index_settings)
client.create_alias('events-previous', previous_event_index_name)

if alert_index_name not in all_indices:
    print "Creating " + alert_index_name
    client.create_index(alert_index_name, index_config=index_settings)
client.create_alias('alerts', alert_index_name)

if weekly_index_alias not in all_indices:
    print "Creating " + weekly_index_alias
    client.create_alias_multiple_indices(weekly_index_alias, [event_index_name, previous_event_index_name])

if state_index_name not in all_indices:
    print "Creating " + state_index_name
    client.create_index(state_index_name, index_config=state_index_settings)

# Wait for kibana service to get ready
total_num_tries = 10
for attempt in range(total_num_tries):
    try:
        if requests.get(kibana_url).ok:
            break
    except Exception:
        pass
    print('Unable to connect to Elasticsearch...retrying')
    sleep(5)
else:
    print('Cannot connect to Kibana after ' + str(total_num_tries) + ' tries, exiting script.')
    exit(1)

# Check if index-patterns already exist
if kibana_index_name in client.get_indices():
    existing_patterns_url = kibana_url + "/api/saved_objects/_find?type=index-pattern&search_fields=title&search=*"
    resp = requests.get(url=existing_patterns_url)
    existing_patterns = json.loads(resp.text)
    if len(existing_patterns['saved_objects']) > 0:
        print("Index patterns already exist, exiting script early")
        sys.exit(0)

# Create index-patterns
kibana_headers = {'content-type': 'application/json', 'kbn-xsrf': 'true'}
index_mappings_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'index_mappings')
listing = os.listdir(index_mappings_path)
for infile in listing:
    json_file_path = os.path.join(index_mappings_path, infile)
    with open(json_file_path) as json_data:
        mapping_data = json.load(json_data)
        index_name = mapping_data['attributes']['title']
        print "Creating {0} index mapping".format(index_name)
        mapping_url = kibana_url + "/api/saved_objects/index-pattern/" + index_name
        resp = requests.post(url=mapping_url, data=json.dumps(mapping_data), headers=kibana_headers)
        if not resp.ok:
            print("Unable to create index mapping: " + resp.text)

# Remove existing default index mapping if it exists
resp = requests.delete(url=kibana_url + "/api/saved_objects/config/" + kibana_version, headers=kibana_headers)
if not resp.ok:
    print("Unable to delete existing default index mapping: {} {}".format(resp.status_code, resp.content))

# Set default index mapping to events-*
data = {
    "attributes": {
        "buildNum": "19548",
        "defaultIndex": "events-*"
    }
}
print("Creating default index pattern for events-*")
resp = requests.post(url=kibana_url + "/api/saved_objects/config/" + kibana_version, data=json.dumps(data), headers=kibana_headers)
if not resp.ok:
    print("Failed to set default index: {} {}".format(resp.status_code, resp.content))

# Check if dashboards already exist
if kibana_index_name in client.get_indices():
    existing_patterns_url = kibana_url + "/api/saved_objects/_find?type=dashboard&search_fields=title&search=*"
    resp = requests.get(url=existing_patterns_url)
    existing_patterns = json.loads(resp.text)
    if len(existing_patterns['saved_objects']) > 0:
        print("Dashboards already exist, exiting script early")
        sys.exit(0)

# Create visualizations/dashboards
dashboards_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'resources')
listing = os.listdir(dashboards_path)
for infile in listing:
    json_file_path = os.path.join(dashboards_path, infile)
    with open(json_file_path) as json_data:
        mapping_data = json.load(json_data)
        mapping_type = mapping_data['type']
        print("Creating {0} {1}".format(
            mapping_data[mapping_type]['title'],
            mapping_type
        ))
        post_data = {
            "attributes": mapping_data[mapping_type]
        }
        # We use the filename as the id of the resource
        resource_name = infile.replace('.json', '')
        kibana_type_url = kibana_url + "/api/saved_objects/" + mapping_type + "/" + resource_name
        requests.post(url=kibana_type_url, data=json.dumps(post_data), headers=kibana_headers)
<|endoftext|>


<commit_before>from mpi4py import MPI
import mpiunittest as unittest

datatypes_c = [
MPI.CHAR, MPI.WCHAR,
MPI.SIGNED_CHAR, MPI.SHORT, MPI.INT, MPI.LONG,
MPI.UNSIGNED_CHAR, MPI.UNSIGNED_SHORT, MPI.UNSIGNED, MPI.UNSIGNED_LONG,
MPI.LONG_LONG, MPI.UNSIGNED_LONG_LONG,
MPI.FLOAT, MPI.DOUBLE, MPI.LONG_DOUBLE,
]
datatypes_c99 = [
MPI.C_BOOL,
MPI.INT8_T, MPI.INT16_T, MPI.INT32_T, MPI.INT64_T,
MPI.UINT8_T, MPI.UINT16_T, MPI.UINT32_T, MPI.UINT64_T,
MPI.C_COMPLEX, MPI.C_FLOAT_COMPLEX,
MPI.C_DOUBLE_COMPLEX, MPI.C_LONG_DOUBLE_COMPLEX,
]
datatypes_f = [
MPI.CHARACTER, MPI.LOGICAL, MPI.INTEGER,
MPI.REAL, MPI.DOUBLE_PRECISION,
MPI.COMPLEX, MPI.DOUBLE_COMPLEX,
]
datatypes_f90 = [
MPI.LOGICAL1, MPI.LOGICAL2, MPI.LOGICAL4, MPI.LOGICAL8,
MPI.INTEGER1, MPI.INTEGER2, MPI.INTEGER4, MPI.INTEGER8, MPI.INTEGER16,
MPI.REAL2, MPI.REAL4, MPI.REAL8, MPI.REAL16,
MPI.COMPLEX4, MPI.COMPLEX8, MPI.COMPLEX16, MPI.COMPLEX32,
]
datatypes_mpi = [
MPI.PACKED, MPI.BYTE, MPI.AINT, MPI.OFFSET,
]

datatypes = []
datatypes += datatypes_c
datatypes += datatypes_c99
datatypes += datatypes_f
datatypes += datatypes_f90
datatypes += datatypes_mpi
datatypes = [t for t in datatypes if t != MPI.DATATYPE_NULL]

combiner_map = {}

class TestDatatype(unittest.TestCase):

    def testGetExtent(self):
        for dtype in datatypes:
            lb, ext = dtype.Get_extent()

    def testGetSize(self):
        for dtype in datatypes:
            size = dtype.Get_size()

    def testGetTrueExtent(self):
        for dtype in datatypes:
            try:
                lb, ext = dtype.Get_true_extent()
            except NotImplementedError:
                return

    def testGetEnvelope(self):
        for dtype in datatypes:
            try:
                envelope = dtype.Get_envelope()
            except NotImplementedError:
                return
            if ('LAM/MPI' == MPI.get_vendor()[0] and
                "COMPLEX" in dtype.name): continue
            ni, na, nd, combiner = envelope
            self.assertEqual(combiner, MPI.COMBINER_NAMED)
            self.assertEqual(ni, 0)
            self.assertEqual(na, 0)
            self.assertEqual(nd, 0)

    def _test_derived_contents(self, oldtype, factory, newtype):
        try:
            envelope = newtype.Get_envelope()
            contents = newtype.Get_contents()
        except NotImplementedError:
            return
        ni, na, nd, combiner = envelope
        i, a, d = contents
        self.assertEqual(ni, len(i))
        self.assertEqual(na, len(a))
        self.assertEqual(nd, len(d))
        self.assertTrue(combiner != MPI.COMBINER_NAMED)
        name = factory.__name__
        NAME = name.replace('Create_', '').upper()
        symbol = getattr(MPI, 'COMBINER_' + NAME)
        if symbol == MPI.UNDEFINED: return
        if combiner_map is None: return
        symbol = combiner_map.get(symbol, symbol)
        if symbol is None: return
        self.assertEqual(symbol, combiner)
        decoded = newtype.decode()
        oldtype, constructor, kargs = decoded
        constructor = 'Create_' + constructor.lower()
        if combiner in [MPI.COMBINER_CONTIGUOUS]:
            # Cython could optimize one-arg methods
            newtype2 = getattr(oldtype, constructor)(kargs['count'])
        else:
            newtype2 = getattr(oldtype, constructor)(**kargs)
        decoded2 = newtype2.decode()
        self.assertEqual(decoded[1], decoded2[1])
        self.assertEqual(decoded[2], decoded2[2])
        newtype2.Free()

    def _test_derived(self, oldtype, factory, *args):
        try:
            if isinstance(oldtype, MPI.Datatype):
                newtype = factory(oldtype, *args)
            else:
                newtype = factory(*args)
        except NotImplementedError:
            return
        self._test_derived_contents(oldtype, factory,  newtype)
        newtype.Commit()
        self._test_derived_contents(oldtype, factory,  newtype)
        newtype.Free()

    def testDup(self):
        for dtype in datatypes:
            factory = MPI.Datatype.Dup
            self._test_derived(dtype, factory)

    def testCreateContiguous(self):
        for dtype in datatypes:
            for count in range(5):
                factory = MPI.Datatype.Create_contiguous
                args = (count, )
                self._test_derived(dtype, factory, *args)

    def testCreateVector(self):
        for dtype in datatypes:
            for count in range(5):
                for blocklength in range(5):
                    for stride in range(5):
                        factory = MPI.Datatype.Create_vector
                        args = (count, blocklength, stride)
                        self._test_derived(dtype, factory, *args)

    def testCreateHvector(self):
        for dtype in datatypes:
            for count in range(5):
                for blocklength in range(5):
                    for stride in range(5):
                        factory = MPI.Datatype.Create_hvector
                        args = (count, blocklength, stride)
                        self._test_derived(dtype, factory, *args)

    def testCreateIndexed(self):
        for dtype in datatypes:
            for block in range(5):
                blocklengths = list(range(block, block+5))
                displacements = [0]
                for b in blocklengths[:-1]:
                    stride = displacements[-1] + b * dtype.extent + 1
                    displacements.append(stride)
                factory = MPI.Datatype.Create_indexed
                args = (blocklengths, displacements)
                self._test_derived(dtype, factory, *args)
                #args = (block, displacements) XXX
                #self._test_derived(dtype, factory, *args)  XXX

    def testCreateIndexedBlock(self):
        for dtype in datatypes:
            for block in range(5):
                blocklengths = list(range(block, block+5))
                displacements = [0]
                for b in blocklengths[:-1]:
                    stride = displacements[-1] + b * dtype.extent + 1
                    displacements.append(stride)
                factory = MPI.Datatype.Create_indexed_block
                args = (block, displacements)
                self._test_derived(dtype, factory, *args)

    def testCreateHindexed(self):
        for dtype in datatypes:
            for block in range(5):
                blocklengths = list(range(block, block+5))
                displacements = [0]
                for b in blocklengths[:-1]:
                    stride = displacements[-1] + b * dtype.extent + 1
                    displacements.append(stride)

                factory = MPI.Datatype.Create_hindexed
                args = (blocklengths, displacements)
                self._test_derived(dtype, factory, *args)
                #args = (block, displacements) XXX
                #self._test_derived(dtype, factory, *args)  XXX

    def testCreateHindexedBlock(self):
        for dtype in datatypes:
            for block in range(5):
                displacements = [0]
                for i in range(5):
                    stride = displacements[-1] + block * dtype.extent + 1
                    displacements.append(stride)
                factory = MPI.Datatype.Create_hindexed_block
                args = (block, displacements)
                self._test_derived(dtype, factory, *args)

    def testCreateStruct(self):
        for dtype1 in datatypes:
            for dtype2 in datatypes:
                dtypes = (dtype1, dtype2)
                blocklengths  = (2, 3)
                displacements = [0]
                for dtype in dtypes[:-1]:
                    stride = displacements[-1] + dtype.extent
                    displacements.append(stride)
                factory = MPI.Datatype.Create_struct
                args = (blocklengths, displacements, dtypes)
                self._test_derived(dtypes, factory, *args)

    def testCreateSubarray(self):
        for dtype in datatypes:
            for ndim in range(1, 5):
                for size in range(1, 5):
                    for subsize in range(1, size):
                        for start in range(size-subsize):
                            for order in [MPI.ORDER_C,
                                          MPI.ORDER_FORTRAN,
                                          MPI.ORDER_F,
                                          ]:
                                sizes = [size] * ndim
                                subsizes = [subsize] * ndim
                                starts = [start] * ndim
                                factory = MPI.Datatype.Create_subarray
                                args = sizes, subsizes, starts, order
                                self._test_derived(dtype, factory, *args)

    def testResized(self):
        for dtype in datatypes:
            for lb in range(-10, 10):
                for extent in range(1, 10):
                    factory = MPI.Datatype.Create_resized
                    args = lb, extent
                    self._test_derived(dtype, factory, *args)

    def testGetSetName(self):
        for dtype in datatypes:
            try:
                name = dtype.Get_name()
                self.assertTrue(name)
                dtype.Set_name(name)
                self.assertEqual(name, dtype.Get_name())
            except NotImplementedError:
                return

    def testCommit(self):
        for dtype in datatypes:
            dtype.Commit()

class TestGetAddress(unittest.TestCase):

    def testGetAddress(self):
        try:
            from array import array
            location = array('i', range(10))
            bufptr, _ = location.buffer_info()
            addr = MPI.Get_address(location)
            self.assertEqual(addr, bufptr)
        except ImportError:
            pass
        try:
            from numpy import asarray
            location = asarray(range(10), dtype='i')
            bufptr, _ = location.__array_interface__['data']
            addr = MPI.Get_address(location)
            self.assertEqual(addr, bufptr)
        except ImportError:
            pass

import sys
_name, _version = MPI.get_vendor()
if _name == 'LAM/MPI':
    combiner_map[MPI.COMBINER_INDEXED_BLOCK] = MPI.COMBINER_INDEXED
elif _name == 'MPICH1':
    combiner_map[MPI.COMBINER_VECTOR]  = None
    combiner_map[MPI.COMBINER_HVECTOR] = None
    combiner_map[MPI.COMBINER_INDEXED] = None
    for t in datatypes_f: datatypes.remove(t)
elif MPI.Get_version() < (2, 0):
    combiner_map = None
if _name == 'Open MPI':
    if _version <= (1, 5, 1):
        for t in datatypes_f90[-4:]:
            if t != MPI.DATATYPE_NULL:
                datatypes.remove(t)
    if 'win' in sys.platform:
        del TestDatatype.testCommit
        del TestDatatype.testDup
        del TestDatatype.testResized

if sys.version_info[0] >=3:
    del TestGetAddress

if __name__ == '__main__':
    unittest.main()
<commit_msg>test: Add more tests for Datatype<commit_after>from mpi4py import MPI
import mpiunittest as unittest

datatypes_c = [
MPI.CHAR, MPI.WCHAR,
MPI.SIGNED_CHAR, MPI.SHORT, MPI.INT, MPI.LONG,
MPI.UNSIGNED_CHAR, MPI.UNSIGNED_SHORT, MPI.UNSIGNED, MPI.UNSIGNED_LONG,
MPI.LONG_LONG, MPI.UNSIGNED_LONG_LONG,
MPI.FLOAT, MPI.DOUBLE, MPI.LONG_DOUBLE,
]
datatypes_c99 = [
MPI.C_BOOL,
MPI.INT8_T, MPI.INT16_T, MPI.INT32_T, MPI.INT64_T,
MPI.UINT8_T, MPI.UINT16_T, MPI.UINT32_T, MPI.UINT64_T,
MPI.C_COMPLEX, MPI.C_FLOAT_COMPLEX,
MPI.C_DOUBLE_COMPLEX, MPI.C_LONG_DOUBLE_COMPLEX,
]
datatypes_f = [
MPI.CHARACTER, MPI.LOGICAL, MPI.INTEGER,
MPI.REAL, MPI.DOUBLE_PRECISION,
MPI.COMPLEX, MPI.DOUBLE_COMPLEX,
]
datatypes_f90 = [
MPI.LOGICAL1, MPI.LOGICAL2, MPI.LOGICAL4, MPI.LOGICAL8,
MPI.INTEGER1, MPI.INTEGER2, MPI.INTEGER4, MPI.INTEGER8, MPI.INTEGER16,
MPI.REAL2, MPI.REAL4, MPI.REAL8, MPI.REAL16,
MPI.COMPLEX4, MPI.COMPLEX8, MPI.COMPLEX16, MPI.COMPLEX32,
]
datatypes_mpi = [
MPI.PACKED, MPI.BYTE, MPI.AINT, MPI.OFFSET,
]

datatypes = []
datatypes += datatypes_c
datatypes += datatypes_c99
datatypes += datatypes_f
datatypes += datatypes_f90
datatypes += datatypes_mpi
datatypes = [t for t in datatypes if t != MPI.DATATYPE_NULL]

combiner_map = {}

class TestDatatype(unittest.TestCase):

    def testBoolEqNe(self):
        for dtype in datatypes:
            self.assertTrue (not not dtype)
            self.assertTrue (dtype == MPI.Datatype(dtype))
            self.assertFalse(dtype != MPI.Datatype(dtype))

    def testGetExtent(self):
        for dtype in datatypes:
            lb, ext = dtype.Get_extent()
            self.assertEqual(dtype.lb, lb)
            self.assertEqual(dtype.ub, lb+ext)
            self.assertEqual(dtype.extent, ext)

    def testGetSize(self):
        for dtype in datatypes:
            size = dtype.Get_size()
            self.assertTrue(dtype.size, size)

    def testGetTrueExtent(self):
        for dtype in datatypes:
            try:
                lb, ext = dtype.Get_true_extent()
                self.assertEqual(dtype.true_lb, lb)
                self.assertEqual(dtype.true_ub, lb+ext)
                self.assertEqual(dtype.true_extent, ext)
            except NotImplementedError:
                return

    def testGetEnvelope(self):
        for dtype in datatypes:
            try:
                envelope = dtype.Get_envelope()
            except NotImplementedError:
                return
            if ('LAM/MPI' == MPI.get_vendor()[0] and
                "COMPLEX" in dtype.name): continue
            ni, na, nd, combiner = envelope
            self.assertEqual(combiner, MPI.COMBINER_NAMED)
            self.assertEqual(ni, 0)
            self.assertEqual(na, 0)
            self.assertEqual(nd, 0)

    def _test_derived_contents(self, oldtype, factory, newtype):
        try:
            envelope = newtype.Get_envelope()
            contents = newtype.Get_contents()
        except NotImplementedError:
            return
        ni, na, nd, combiner = envelope
        i, a, d = contents
        self.assertEqual(ni, len(i))
        self.assertEqual(na, len(a))
        self.assertEqual(nd, len(d))
        self.assertTrue(combiner != MPI.COMBINER_NAMED)
        name = factory.__name__
        NAME = name.replace('Create_', '').upper()
        symbol = getattr(MPI, 'COMBINER_' + NAME)
        if symbol == MPI.UNDEFINED: return
        if combiner_map is None: return
        symbol = combiner_map.get(symbol, symbol)
        if symbol is None: return
        self.assertEqual(symbol, combiner)
        decoded = newtype.decode()
        oldtype, constructor, kargs = decoded
        constructor = 'Create_' + constructor.lower()
        if combiner in [MPI.COMBINER_CONTIGUOUS]:
            # Cython could optimize one-arg methods
            newtype2 = getattr(oldtype, constructor)(kargs['count'])
        else:
            newtype2 = getattr(oldtype, constructor)(**kargs)
        decoded2 = newtype2.decode()
        self.assertEqual(decoded[1], decoded2[1])
        self.assertEqual(decoded[2], decoded2[2])
        newtype2.Free()

    def _test_derived(self, oldtype, factory, *args):
        try:
            if isinstance(oldtype, MPI.Datatype):
                newtype = factory(oldtype, *args)
            else:
                newtype = factory(*args)
        except NotImplementedError:
            return
        self._test_derived_contents(oldtype, factory,  newtype)
        newtype.Commit()
        self._test_derived_contents(oldtype, factory,  newtype)
        newtype.Free()

    def testDup(self):
        for dtype in datatypes:
            factory = MPI.Datatype.Dup
            self._test_derived(dtype, factory)

    def testCreateContiguous(self):
        for dtype in datatypes:
            for count in range(5):
                factory = MPI.Datatype.Create_contiguous
                args = (count, )
                self._test_derived(dtype, factory, *args)

    def testCreateVector(self):
        for dtype in datatypes:
            for count in range(5):
                for blocklength in range(5):
                    for stride in range(5):
                        factory = MPI.Datatype.Create_vector
                        args = (count, blocklength, stride)
                        self._test_derived(dtype, factory, *args)

    def testCreateHvector(self):
        for dtype in datatypes:
            for count in range(5):
                for blocklength in range(5):
                    for stride in range(5):
                        factory = MPI.Datatype.Create_hvector
                        args = (count, blocklength, stride)
                        self._test_derived(dtype, factory, *args)

    def testCreateIndexed(self):
        for dtype in datatypes:
            for block in range(5):
                blocklengths = list(range(block, block+5))
                displacements = [0]
                for b in blocklengths[:-1]:
                    stride = displacements[-1] + b * dtype.extent + 1
                    displacements.append(stride)
                factory = MPI.Datatype.Create_indexed
                args = (blocklengths, displacements)
                self._test_derived(dtype, factory, *args)
                #args = (block, displacements) XXX
                #self._test_derived(dtype, factory, *args)  XXX

    def testCreateIndexedBlock(self):
        for dtype in datatypes:
            for block in range(5):
                blocklengths = list(range(block, block+5))
                displacements = [0]
                for b in blocklengths[:-1]:
                    stride = displacements[-1] + b * dtype.extent + 1
                    displacements.append(stride)
                factory = MPI.Datatype.Create_indexed_block
                args = (block, displacements)
                self._test_derived(dtype, factory, *args)

    def testCreateHindexed(self):
        for dtype in datatypes:
            for block in range(5):
                blocklengths = list(range(block, block+5))
                displacements = [0]
                for b in blocklengths[:-1]:
                    stride = displacements[-1] + b * dtype.extent + 1
                    displacements.append(stride)

                factory = MPI.Datatype.Create_hindexed
                args = (blocklengths, displacements)
                self._test_derived(dtype, factory, *args)
                #args = (block, displacements) XXX
                #self._test_derived(dtype, factory, *args)  XXX

    def testCreateHindexedBlock(self):
        for dtype in datatypes:
            for block in range(5):
                displacements = [0]
                for i in range(5):
                    stride = displacements[-1] + block * dtype.extent + 1
                    displacements.append(stride)
                factory = MPI.Datatype.Create_hindexed_block
                args = (block, displacements)
                self._test_derived(dtype, factory, *args)

    def testCreateStruct(self):
        for dtype1 in datatypes:
            for dtype2 in datatypes:
                dtypes = (dtype1, dtype2)
                blocklengths  = (2, 3)
                displacements = [0]
                for dtype in dtypes[:-1]:
                    stride = displacements[-1] + dtype.extent
                    displacements.append(stride)
                factory = MPI.Datatype.Create_struct
                args = (blocklengths, displacements, dtypes)
                self._test_derived(dtypes, factory, *args)

    def testCreateSubarray(self):
        for dtype in datatypes:
            for ndim in range(1, 5):
                for size in range(1, 5):
                    for subsize in range(1, size):
                        for start in range(size-subsize):
                            for order in [MPI.ORDER_C,
                                          MPI.ORDER_FORTRAN,
                                          MPI.ORDER_F,
                                          ]:
                                sizes = [size] * ndim
                                subsizes = [subsize] * ndim
                                starts = [start] * ndim
                                factory = MPI.Datatype.Create_subarray
                                args = sizes, subsizes, starts, order
                                self._test_derived(dtype, factory, *args)

    def testResized(self):
        for dtype in datatypes:
            for lb in range(-10, 10):
                for extent in range(1, 10):
                    factory = MPI.Datatype.Create_resized
                    args = lb, extent
                    self._test_derived(dtype, factory, *args)

    def testGetSetName(self):
        for dtype in datatypes:
            try:
                name = dtype.Get_name()
                self.assertTrue(name)
                dtype.Set_name(name)
                self.assertEqual(name, dtype.Get_name())
            except NotImplementedError:
                return

    def testCommit(self):
        for dtype in datatypes:
            dtype.Commit()

class TestGetAddress(unittest.TestCase):

    def testGetAddress(self):
        try:
            from array import array
            location = array('i', range(10))
            bufptr, _ = location.buffer_info()
            addr = MPI.Get_address(location)
            self.assertEqual(addr, bufptr)
        except ImportError:
            pass
        try:
            from numpy import asarray
            location = asarray(range(10), dtype='i')
            bufptr, _ = location.__array_interface__['data']
            addr = MPI.Get_address(location)
            self.assertEqual(addr, bufptr)
        except ImportError:
            pass

import sys
_name, _version = MPI.get_vendor()
if _name == 'LAM/MPI':
    combiner_map[MPI.COMBINER_INDEXED_BLOCK] = MPI.COMBINER_INDEXED
elif _name == 'MPICH1':
    combiner_map[MPI.COMBINER_VECTOR]  = None
    combiner_map[MPI.COMBINER_HVECTOR] = None
    combiner_map[MPI.COMBINER_INDEXED] = None
    for t in datatypes_f: datatypes.remove(t)
elif MPI.Get_version() < (2, 0):
    combiner_map = None
if _name == 'Open MPI':
    if _version <= (1, 5, 1):
        for t in datatypes_f90[-4:]:
            if t != MPI.DATATYPE_NULL:
                datatypes.remove(t)
    if 'win' in sys.platform:
        del TestDatatype.testCommit
        del TestDatatype.testDup
        del TestDatatype.testResized

if sys.version_info[0] >=3:
    del TestGetAddress

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import lsst.log
Log = lsst.log.Log()
Log.setLevel(lsst.log.ERROR)

from . import imtools
from . import stats
from . import synths
from . import cattools
from .parser import parse_args
from .synths import SynthFactory
from .stats import get_clipped_sig_task
from .run import run
from .primitives import *
from .viewer import Viewer
from .config import Config
from .utils import io, pixscale, get_group_patches
<commit_msg>check for lsst.log<commit_after>try:
    import lsst.log
    Log = lsst.log.Log()
    Log.setLevel(lsst.log.ERROR)
except ImportError:
    pass

from . import imtools
from . import stats
from . import synths
from . import cattools
from .parser import parse_args
from .synths import SynthFactory
from .stats import get_clipped_sig_task
from .run import run
from .primitives import *
from .viewer import Viewer
from .config import Config
from .utils import io, pixscale, get_group_patches
<|endoftext|>


<commit_before># from https://github.com/timothycrosley/hug
# install pip3 install hug --upgrade
# A basic (single function) API written using Hug
# example call http://127.0.0.1:8000/redis_call?ape=123456&rname=phrase
import hug
import redis
# import MySQLdb
try:
    r = redis.StrictRedis(host='127.0.0.1',port=6379)
except:
    print("\n warning! ----- redis not running -----\n\n")
@hug.get('/happy_birthday')

def happy_birthday(name, age:hug.types.number=1):
    """Says happy birthday to a user"""
    return "Happy {age} Birthday {name}!".format(**locals())

@hug.get('/redis_call')
def a_redis_call(rname, ape=1):
    if ape == 1:
        return "no valid api key specified, nice try though".format(**locals())
    if r.sismember('ape',ape) != 1:
        return "no valid api key specified, nice try though".format(**locals())
    else:
        coolness = r.get(rname).decode('utf8')
        r.decr(ape)
        numleft = int(r.get(ape))
        return "Authenticated using api key {ape}. You have {numleft} queries left. This is the {rname} value you requested: {coolness}".format(**locals())
<commit_msg>updated documentation on hug api example<commit_after># from https://github.com/timothycrosley/hug
# install pip3 install hug --upgrade
# A basic (single function) API written using Hug
# Make sure you have redis installed via pip and redis-cli can connect
# example add data first: http://127.0.0.1:8000/redis_add?ape=123456&rname=phrase
# example call http://127.0.0.1:8000/redis_call?ape=123456&rname=phrase
import hug
import redis
# import MySQLdb
try:
    r = redis.StrictRedis(host='127.0.0.1', port=6379)
except:
    print("\n warning! ----- redis not running -----\n\n")

@hug.get('/happy_birthday')
def happy_birthday(name, age: hug.types.number = 1):
    """Says happy birthday to a user"""
    return "Happy {age} Birthday {name}!".format(**locals())

@hug.get('/redis_call')
def a_redis_call(rname, ape=1):
    if ape == 1:
        return "no valid api key specified, nice try though".format(**locals())
    if r.sismember('ape', ape) != 1:
        return "no valid api key specified, nice try though".format(**locals())
    else:
        coolness = r.get(rname).decode('utf8')
        r.decr(ape)
        numleft = r.get(str(ape))
        print(numleft)
        return "Authenticated using api key {ape}. You have {numleft} queries left. This is the {rname} value you requested: {coolness}".format(**locals())

@hug.get('/redis_add')
def add_redis(rname, ape=1):
    r.sadd('ape', int(ape))
    r.set(ape, 1000)
    r.set(rname, 'a nice value here')
    return "added successfully".format(**locals())
<|endoftext|>


<commit_before># Copyright 2014-2016 The ODL development group
#
# This file is part of ODL.
#
# ODL is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ODL is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with ODL.  If not, see <http://www.gnu.org/licenses/>.

"""Tests that all examples run at all.

Running this file causes all examples to be run, and any exception will give
a FAILED result. All other results are considered PASSED.

Plots are displayed while the examples are run, but are closed after each
example. A user can, if he/she wants inspect the plots to further see if the
examples are OK.

This package assumes that all dependencies are installed.
"""

# Imports for common Python 2/3 codebase
from __future__ import print_function, division, absolute_import
from future import standard_library
standard_library.install_aliases()

import os
import imp
import pytest
import matplotlib
matplotlib.use('Agg')  # To avoid the backend freezing
import matplotlib.pyplot as plt

# Make a fixture for all examples
this_file_path = os.path.dirname(os.path.abspath(__file__))
example_ids = []
example_params = []
for dirpath, dirnames, filenames in os.walk(this_file_path + "/../examples/"):
    for filename in [f for f in filenames if f.endswith(".py") and
                     not f.startswith('__init__')]:
        example_params.append(os.path.join(dirpath, filename))
        example_ids.append(filename[:-3])  # skip .py

@pytest.fixture(scope="module", ids=example_ids, params=example_params)
def example(request):
    return request.param

@pytest.mark.skipif("not pytest.config.getoption('--examples')",
                    reason='Need --examples option to run')
def test_example(example):
    imp.load_source('tmp', example)
    plt.close("all")

if __name__ == '__main__':
    pytest.main(str(__file__.replace('\\', '/')) + ' -v --examples')
<commit_msg>TST: fence matplotlib import for CI testing w/o it<commit_after># Copyright 2014-2016 The ODL development group
#
# This file is part of ODL.
#
# ODL is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ODL is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with ODL.  If not, see <http://www.gnu.org/licenses/>.

"""Tests that all examples run at all.

Running this file causes all examples to be run, and any exception will give
a FAILED result. All other results are considered PASSED.

Plots are displayed while the examples are run, but are closed after each
example. A user can, if he/she wants inspect the plots to further see if the
examples are OK.

This package assumes that all dependencies are installed.
"""

# Imports for common Python 2/3 codebase
from __future__ import print_function, division, absolute_import
from future import standard_library
standard_library.install_aliases()

import os
import imp
import pytest
try:
    import matplotlib
    matplotlib.use('Agg')  # To avoid the backend freezing
    import matplotlib.pyplot as plt
except ImportError:
    pass

# Make a fixture for all examples
this_file_path = os.path.dirname(os.path.abspath(__file__))
example_ids = []
example_params = []
for dirpath, dirnames, filenames in os.walk(this_file_path + "/../examples/"):
    for filename in [f for f in filenames if f.endswith(".py") and
                     not f.startswith('__init__')]:
        example_params.append(os.path.join(dirpath, filename))
        example_ids.append(filename[:-3])  # skip .py

@pytest.fixture(scope="module", ids=example_ids, params=example_params)
def example(request):
    return request.param

@pytest.mark.skipif("not pytest.config.getoption('--examples')",
                    reason='Need --examples option to run')
def test_example(example):
    imp.load_source('tmp', example)
    plt.close("all")

if __name__ == '__main__':
    pytest.main(str(__file__.replace('\\', '/')) + ' -v --examples')
<|endoftext|>


<commit_before>VERSION = '0.1.0'

INT = 'Int'
BOOL = 'Bool'
types = [INT, BOOL]

LITERAL_STRINGS = [
    'True',
    'False',
    *[str(i) for i in range(10)],
    *['function{}'.format(i) for i in range(4)],
    '::',
    '->',
    'Bool',
    'Int',
    *[chr(ord('a') + i) for i in range(8)],
    '(',
    ')',
    'div',
    'mod',
    'max',
    'min',
    '+',
    '-',
    '*',
    '==',
    '/=',
    '>',
    '>=',
    '<',
    '<=',
    '||',
    '&&',
    '||',
    'not',
    'fromEnum',
    'module',
    'where',
    'Generated',
    ','
]

TOKENISER = {k: i for (k, i) in zip(LITERAL_STRINGS, range(len(LITERAL_STRINGS)))}
<commit_msg>Push new version.<commit_after>VERSION = '0.1.1'

INT = 'Int'
BOOL = 'Bool'
types = [INT, BOOL]

LITERAL_STRINGS = [
    'True',
    'False',
    *[str(i) for i in range(10)],
    *['function{}'.format(i) for i in range(4)],
    '::',
    '->',
    'Bool',
    'Int',
    *[chr(ord('a') + i) for i in range(8)],
    '(',
    ')',
    'div',
    'mod',
    'max',
    'min',
    '+',
    '-',
    '*',
    '==',
    '/=',
    '>',
    '>=',
    '<',
    '<=',
    '||',
    '&&',
    '||',
    'not',
    'fromEnum',
    'module',
    'where',
    'Generated',
    ','
]

TOKENISER = {k: i for (k, i) in zip(LITERAL_STRINGS, range(len(LITERAL_STRINGS)))}
<|endoftext|>