<commit_before>##########################################################################
# Copyright 2013-2021 Aerospike, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##########################################################################
'''
Helper functions to create HyperLogLog operation dictionary arguments for
the :meth:`aerospike.operate` and :meth:`aerospike.operate_ordered` methods of the aerospike client.
HyperLogLog bins and operations allow for your application to form fast, reasonable approximations
of members in the union or intersection between multiple HyperLogLog bins.
HyperLogLog’s estimates are a balance between complete accuracy and efficient savings
in space and speed in dealing with extremely large datasets.

    .. note:: HyperLogLog operations require server version >= 4.9.0

    .. seealso:: `HyperLogLog (Data Type) more info. <https://www.aerospike.com/docs/guide/hyperloglog.html>`_.

Example::

    import sys

    import aerospike
    from aerospike import exception as ex
    from aerospike_helpers.operations import hll_operations as hll_ops
    from aerospike_helpers.operations import operations

    TEST_NS = "test"
    TEST_SET = "demo"
    NUM_INDEX_BITS = 12
    NUM_MH_BITS = 24

    # Configure the client.
    config = {"hosts": [("127.0.0.1", 3000)]}

    # Create a client and connect it to the cluster.
    try:
        client = aerospike.client(config).connect()
    except ex.ClientError as e:
        print("Error: {0} [{1}]".format(e.msg, e.code))
        sys.exit(1)

    # Add HLL bins.
    customers = ["Amy", "Farnsworth", "Scruffy"]
    customer_record_keys = [
        (TEST_NS, TEST_SET, "Amy"),
        (TEST_NS, TEST_SET, "Farnsworth"),
        (TEST_NS, TEST_SET, "Scruffy"),
    ]
    items_viewed = [
        ("item%s" % str(i) for i in range(0, 500)),
        ("item%s" % str(i) for i in range(0, 750)),
        ("item%s" % str(i) for i in range(250, 1000)),
    ]

    for customer, key, items in zip(customers, customer_record_keys, items_viewed):
        ops = [
            operations.write("name", customer),
            hll_ops.hll_add("viewed", list(items), NUM_INDEX_BITS, NUM_MH_BITS),
        ]

        try:
            client.operate(key, ops)
        except ex.ClientError as e:
            print("Error: {0} [{1}]".format(e.msg, e.code))
            sys.exit(1)

    # Find out how many items viewed Amy, Farnsworth, and Scruffy have in common.
    Farnsworth_viewed = client.get(customer_record_keys[1])[2]["viewed"]
    Scruffy_viewed = client.get(customer_record_keys[2])[2]["viewed"]
    viewed = [Farnsworth_viewed, Scruffy_viewed]
    ops = [hll_ops.hll_get_intersect_count("viewed", viewed)]

    try:
        _, _, res = client.operate(customer_record_keys[0], ops)
    except ex.ClientError as e:
        print("Error: {0} [{1}]".format(e.msg, e.code))
        sys.exit(1)

    print(
        "Estimated items viewed intersection: %d."
        % res["viewed"]
    )
    print("Actual intersection: 250.\\n")

    # Find out how many unique products Amy, Farnsworth, and Scruffy have viewed.
    Farnsworth_viewed = client.get(customer_record_keys[1])[2]["viewed"]
    Scruffy_viewed = client.get(customer_record_keys[2])[2]["viewed"]
    viewed = [Farnsworth_viewed, Scruffy_viewed]
    ops = [hll_ops.hll_get_union_count("viewed", viewed)]

    try:
        _, _, res = client.operate(customer_record_keys[0], ops)
    except ex.ClientError as e:
        print("Error: {0} [{1}]".format(e.msg, e.code))
        sys.exit(1)

    print(
        "Estimated items viewed union: %d."
        % res["viewed"]
    )
    print("Actual union: 1000.\\n")

    # Find the similarity of Amy, Farnsworth, and Scruffy's product views.
    Farnsworth_viewed = client.get(customer_record_keys[1])[2]["viewed"]
    Scruffy_viewed = client.get(customer_record_keys[2])[2]["viewed"]
    viewed = [Farnsworth_viewed, Scruffy_viewed]
    ops = [hll_ops.hll_get_similarity("viewed", viewed)]

    try:
        _, _, res = client.operate(customer_record_keys[0], ops)
    except ex.ClientError as e:
        print("Error: {0} [{1}]".format(e.msg, e.code))
        sys.exit(1)

    print(
        "Estimated items viewed similarity: %f%%."
        % (res["viewed"] * 100)
    )
    print("Actual similarity: 25%.")

    """
    Expected output:
    Estimated items viewed intersection: 235.
    Actual intersection: 250.

    Estimated items viewed union: 922.
    Actual union: 1000.

    Estimated items viewed similarity: 25.488069%.
    Actual similarity: 25%.
    """

'''

import aerospike

OP_KEY = "op"
BIN_KEY = "bin"
HLL_POLICY_KEY = "hll_policy"
INDEX_BIT_COUNT_KEY = "index_bit_count"
MH_BIT_COUNT_KEY = "mh_bit_count"
VALUE_LIST_KEY = "value_list"

def hll_add(bin_name: str, values, index_bit_count=None, mh_bit_count=None, policy=None):
    """Creates a hll_add operation to be used with :meth:`aerospike.operate` or :meth:`aerospike.operate_ordered`.

    Server will add the values to the hll bin.
    If the HLL bin does not exist, it will be created with index_bit_count and/or mh_bit_count if they have been supplied.

    Args:
        bin_name (str): The name of the bin to be operated on.
        values: The values to be added to the HLL set.
        index_bit_count: An optional number of index bits. Must be bewtween 4 and 16 inclusive.
        mh_bit_count: An optional number of min hash bits. Must be bewtween 4 and 58 inclusive.
        policy (dict): An optional dictionary of :ref:`HyperLogLog policies <aerospike_hll_policies>`.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_ADD,
        BIN_KEY: bin_name,
        VALUE_LIST_KEY: values,
        INDEX_BIT_COUNT_KEY: -1 if index_bit_count is None else index_bit_count,
        MH_BIT_COUNT_KEY: -1 if mh_bit_count is None else mh_bit_count
    }

    if policy:
        op_dict[HLL_POLICY_KEY] = policy

    return op_dict

def hll_describe(bin_name):
    """Creates a hll_describe operation to be used with :meth:`aerospike.operate` or :meth:`aerospike.operate_ordered`.

    Server returns index and minhash bit counts used to create HLL bin in a list of integers.
    The list size is 2.

    Args:
        bin_name (str): The name of the bin to be operated on.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_DESCRIBE,
        BIN_KEY: bin_name,
    }

    return op_dict

def hll_fold(bin_name: str, index_bit_count):
    """Creates a hll_fold operation to be used with :meth:`aerospike.operate` or :meth:`aerospike.operate_ordered`.

    Servers folds index_bit_count to the specified value.
    This can only be applied when minhash bit count on the HLL bin is 0.
    Server does not return a value.

    Args:
        bin_name (str): The name of the bin to be operated on.
        index_bit_count: number of index bits. Must be bewtween 4 and 16 inclusive.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_FOLD,
        BIN_KEY: bin_name,
        INDEX_BIT_COUNT_KEY: index_bit_count
    }

    return op_dict

def hll_get_count(bin_name):
    """Creates a hll_get_count operation to be used with :meth:`aerospike.operate` or :meth:`aerospike.operate_ordered`.

    Server returns estimated count of elements in the HLL bin.

    Args:
        bin_name (str): The name of the bin to be operated on.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_GET_COUNT,
        BIN_KEY: bin_name,
    }

    return op_dict

def hll_get_intersect_count(bin_name: str, hll_list):
    """Creates a hll_get_intersect_count operation to be used with :meth:`aerospike.operate` or :meth:`aerospike.operate_ordered`.

    Server returns estimate of elements that would be contained by the intersection of these HLL objects.

    Args:
        bin_name (str): The name of the bin to be operated on.
        hll_list (list): The HLLs to be intersected.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_GET_INTERSECT_COUNT,
        BIN_KEY: bin_name,
        VALUE_LIST_KEY: hll_list
    }

    return op_dict

def hll_get_similarity(bin_name: str, hll_list):
    """Creates a hll_get_similarity operation to be used with :meth:`aerospike.operate` or :meth:`aerospike.operate_ordered`.

    Server returns estimated similarity of the HLL objects.
    Server returns a float.

    Args:
        bin_name (str): The name of the bin to be operated on.
        hll_list (list): The HLLs used for similarity estimation.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_GET_SIMILARITY,
        BIN_KEY: bin_name,
        VALUE_LIST_KEY: hll_list
    }

    return op_dict

def hll_get_union(bin_name: str, hll_list):
    """Creates a hll_get_union operation to be used with :meth:`aerospike.operate` or :meth:`aerospike.operate_ordered`.

    Server returns an HLL object that is the union of all specified HLL objects
    in hll_list with the HLL bin.

    Args:
        bin_name (str): The name of the bin to be operated on.
        hll_list (list): The HLLs to be unioned.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_GET_UNION,
        BIN_KEY: bin_name,
        VALUE_LIST_KEY: hll_list
    }

    return op_dict

def hll_get_union_count(bin_name: str, hll_list):
    """Creates a hll_get_union_count operation to be used with :meth:`aerospike.operate` or :meth:`aerospike.operate_ordered`.

    Server returns the estimated count of elements that would be contained by the union of all specified HLL objects
    in the list with the HLL bin.

    Args:
        bin_name (str): The name of the bin to be operated on.
        hll_list (list): The HLLs to be unioned.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_GET_UNION_COUNT,
        BIN_KEY: bin_name,
        VALUE_LIST_KEY: hll_list
    }

    return op_dict

def hll_init(bin_name: str, index_bit_count=None, mh_bit_count=None, policy=None):
    """Creates a hll_init operation to be used with :meth:`aerospike.operate` or :meth:`aerospike.operate_ordered`.

    Server creates a new HLL or resets an existing HLL.
    If index_bit_count and mh_bit_count are None, an existing HLL bin will be reset but retain its configuration.
    If 1 of index_bit_count or mh_bit_count are set,
    an existing HLL bin will set that config and retain its current value for the unset config.
    If the HLL bin does not exist, index_bit_count is required to create it, mh_bit_count is optional.
    Server does not return a value.

    Args:
        bin_name (str): The name of the bin to be operated on.
        index_bit_count: An optional number of index bits. Must be bewtween 4 and 16 inclusive.
        mh_bit_count: An optional number of min hash bits. Must be bewtween 4 and 58 inclusive.
        policy (dict): An optional dictionary of :ref:`HyperLogLog policies <aerospike_hll_policies>`.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_INIT,
        BIN_KEY: bin_name,
        INDEX_BIT_COUNT_KEY: -1 if index_bit_count is None else index_bit_count,
        MH_BIT_COUNT_KEY: -1 if mh_bit_count is None else mh_bit_count
    }

    if policy:
        op_dict[HLL_POLICY_KEY] = policy

    return op_dict

def hll_refresh_count(bin_name: str):
    """Creates a hll_refresh_count operation to be used with :meth:`aerospike.operate` or :meth:`aerospike.operate_ordered`.

    Server updates the cached count if it is stale.
    Server returns the count.

    Args:
        bin_name (str): The name of the bin to be operated on.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_REFRESH_COUNT,
        BIN_KEY: bin_name,
    }

    return op_dict

def hll_set_union(bin_name: str, hll_list, policy=None):
    """Creates a hll_set_union operation to be used with :meth:`aerospike.operate` or :meth:`aerospike.operate_ordered`.

    Server sets the union of all specified HLL objects with the HLL bin.
    Server returns nothing.

    Args:
        bin_name (str): The name of the bin to be operated on.
        hll_list (list): The HLLs who's union will be set.
        policy (dict): An optional dictionary of :ref:`HyperLogLog policies <aerospike_hll_policies>`.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_SET_UNION,
        BIN_KEY: bin_name,
        VALUE_LIST_KEY: hll_list
    }

    if policy:
        op_dict[HLL_POLICY_KEY] = policy

    return op_dict
<commit_msg>Docs: revise hll docs and examples<commit_after>##########################################################################
# Copyright 2013-2021 Aerospike, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##########################################################################
'''
Helper functions to create HyperLogLog operation dictionary arguments for
the :meth:`aerospike.Client.operate` and :meth:`aerospike.Client.operate_ordered` methods of the aerospike client.
HyperLogLog bins and operations allow for your application to form fast, reasonable approximations
of members in the union or intersection between multiple HyperLogLog bins.
HyperLogLog’s estimates are a balance between complete accuracy and efficient savings
in space and speed in dealing with extremely large datasets.

    .. note:: HyperLogLog operations require server version >= 4.9.0

    .. seealso:: `HyperLogLog (Data Type) more info. <https://www.aerospike.com/docs/guide/hyperloglog.html>`_.

Example::

    import aerospike
    from aerospike_helpers.operations import hll_operations as hll_ops
    from aerospike_helpers.operations import operations

    NUM_INDEX_BITS = 12
    NUM_MH_BITS = 24

    # Configure the client.
    config = {"hosts": [("127.0.0.1", 3000)]}
    # Create a client and connect it to the cluster.
    client = aerospike.client(config).connect()

    # Create customer keys
    TEST_NS = "test"
    TEST_SET = "demo"
    customerNames = ["Amy", "Farnsworth", "Scruffy"]
    keys = []
    for customer in customerNames:
        keys.append((TEST_NS, TEST_SET, customer))

    itemsViewedPerCustomer = [
        # [item1, item2, ... item500]
        list("item%s" % str(i) for i in range(0, 500)), # Amy
        list("item%s" % str(i) for i in range(0, 750)), # Farnsworth
        list("item%s" % str(i) for i in range(250, 1000)), # Scruffy
    ]

    for key, itemsViewed in zip(keys, itemsViewedPerCustomer):
        customerName = key[2]
        ops = [
            operations.write("name", customerName),
            hll_ops.hll_add("viewed", itemsViewed, NUM_INDEX_BITS, NUM_MH_BITS),
        ]
        client.operate(key, ops)

    # Find out how many items viewed Amy, Farnsworth, and Scruffy have in common.
    farnsworthRecord = client.get(keys[1])
    scruffyRecord = client.get(keys[2])
    farnsworthViewedItems = farnsworthRecord[2]["viewed"]
    scruffyViewedItems = scruffyRecord[2]["viewed"]
    viewed = [farnsworthViewedItems, scruffyViewedItems]
    ops = [
        hll_ops.hll_get_intersect_count("viewed", viewed)
    ]
    # Pass in Amy's key
    _, _, res = client.operate(keys[0], ops)
    print("Estimated items viewed intersection:", res["viewed"])
    # Estimated items viewed intersection: 251
    # Actual intersection: 250

    # Find out how many unique products Amy, Farnsworth, and Scruffy have viewed.
    ops = [hll_ops.hll_get_union_count("viewed", viewed)]
    _, _, res = client.operate(keys[0], ops)

    print("Estimated items viewed union:", res["viewed"])
    # Estimated items viewed union: 1010
    # Actual union: 1000

    # Find the similarity of Amy, Farnsworth, and Scruffy's product views.
    ops = [hll_ops.hll_get_similarity("viewed", viewed)]
    _, _, res = client.operate(keys[0], ops)

    print("Estimated items viewed similarity: %f%%" % (res["viewed"] * 100))
    # Estimated items viewed similarity: 24.888393%
    # Actual similarity: 25%

'''

import aerospike

OP_KEY = "op"
BIN_KEY = "bin"
HLL_POLICY_KEY = "hll_policy"
INDEX_BIT_COUNT_KEY = "index_bit_count"
MH_BIT_COUNT_KEY = "mh_bit_count"
VALUE_LIST_KEY = "value_list"

def hll_add(bin_name: str, values, index_bit_count=None, mh_bit_count=None, policy=None):
    """Creates a hll_add operation.

    Server will add the values to the hll bin.
    If the HLL bin does not exist, it will be created with index_bit_count and/or mh_bit_count if they have been supplied.

    Returns a dictionary to be used with :meth:`aerospike.Client.operate` and :meth:`aerospike.Client.operate_ordered`.

    Args:
        bin_name (str): The name of the bin to be operated on.
        values: The values to be added to the HLL set.
        index_bit_count: An optional number of index bits. Must be bewtween 4 and 16 inclusive.
        mh_bit_count: An optional number of min hash bits. Must be bewtween 4 and 58 inclusive.
        policy (dict): An optional dictionary of :ref:`HyperLogLog policies <aerospike_hll_policies>`.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_ADD,
        BIN_KEY: bin_name,
        VALUE_LIST_KEY: values,
        INDEX_BIT_COUNT_KEY: -1 if index_bit_count is None else index_bit_count,
        MH_BIT_COUNT_KEY: -1 if mh_bit_count is None else mh_bit_count
    }

    if policy:
        op_dict[HLL_POLICY_KEY] = policy

    return op_dict

def hll_describe(bin_name):
    """Creates a hll_describe operation.

    Server returns index and minhash bit counts used to create HLL bin in a list of integers.
    The list size is 2.

    Returns a dictionary to be used with :meth:`aerospike.Client.operate` and :meth:`aerospike.Client.operate_ordered`.

    Args:
        bin_name (str): The name of the bin to be operated on.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_DESCRIBE,
        BIN_KEY: bin_name,
    }

    return op_dict

def hll_fold(bin_name: str, index_bit_count):
    """Creates a hll_fold operation.

    Servers folds index_bit_count to the specified value.
    This can only be applied when minhash bit count on the HLL bin is 0.
    Server does not return a value.

    Returns a dictionary to be used with :meth:`aerospike.Client.operate` and :meth:`aerospike.Client.operate_ordered`.

    Args:
        bin_name (str): The name of the bin to be operated on.
        index_bit_count: number of index bits. Must be bewtween 4 and 16 inclusive.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_FOLD,
        BIN_KEY: bin_name,
        INDEX_BIT_COUNT_KEY: index_bit_count
    }

    return op_dict

def hll_get_count(bin_name):
    """Creates a hll_get_count operation.

    Server returns estimated count of elements in the HLL bin.

    Returns a dictionary to be used with :meth:`aerospike.Client.operate` and :meth:`aerospike.Client.operate_ordered`.

    Args:
        bin_name (str): The name of the bin to be operated on.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_GET_COUNT,
        BIN_KEY: bin_name,
    }

    return op_dict

def hll_get_intersect_count(bin_name: str, hll_list):
    """Creates a hll_get_intersect_count operation.

    Server returns estimate of elements that would be contained by the intersection of these HLL objects.

    Returns a dictionary to be used with :meth:`aerospike.Client.operate` and :meth:`aerospike.Client.operate_ordered`.

    Args:
        bin_name (str): The name of the bin to be operated on.
        hll_list (list): The HLLs to be intersected.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_GET_INTERSECT_COUNT,
        BIN_KEY: bin_name,
        VALUE_LIST_KEY: hll_list
    }

    return op_dict

def hll_get_similarity(bin_name: str, hll_list):
    """Creates a hll_get_similarity operation.

    Server returns estimated similarity of the HLL objects.
    Server returns a float.

    Returns a dictionary to be used with :meth:`aerospike.Client.operate` and :meth:`aerospike.Client.operate_ordered`.

    Args:
        bin_name (str): The name of the bin to be operated on.
        hll_list (list): The HLLs used for similarity estimation.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_GET_SIMILARITY,
        BIN_KEY: bin_name,
        VALUE_LIST_KEY: hll_list
    }

    return op_dict

def hll_get_union(bin_name: str, hll_list):
    """Creates a hll_get_union operation.

    Server returns an HLL object that is the union of all specified HLL objects
    in hll_list with the HLL bin.

    Returns a dictionary to be used with :meth:`aerospike.Client.operate` and :meth:`aerospike.Client.operate_ordered`.

    Args:
        bin_name (str): The name of the bin to be operated on.
        hll_list (list): The HLLs to be unioned.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_GET_UNION,
        BIN_KEY: bin_name,
        VALUE_LIST_KEY: hll_list
    }

    return op_dict

def hll_get_union_count(bin_name: str, hll_list):
    """Creates a hll_get_union_count operation.

    Server returns the estimated count of elements that would be contained by the union of all specified HLL objects
    in the list with the HLL bin.

    Returns a dictionary to be used with :meth:`aerospike.Client.operate` and :meth:`aerospike.Client.operate_ordered`.

    Args:
        bin_name (str): The name of the bin to be operated on.
        hll_list (list): The HLLs to be unioned.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_GET_UNION_COUNT,
        BIN_KEY: bin_name,
        VALUE_LIST_KEY: hll_list
    }

    return op_dict

def hll_init(bin_name: str, index_bit_count=None, mh_bit_count=None, policy=None):
    """Creates a hll_init operation.

    Server creates a new HLL or resets an existing HLL.
    If index_bit_count and mh_bit_count are None, an existing HLL bin will be reset but retain its configuration.
    If 1 of index_bit_count or mh_bit_count are set,
    an existing HLL bin will set that config and retain its current value for the unset config.
    If the HLL bin does not exist, index_bit_count is required to create it, mh_bit_count is optional.
    Server does not return a value.

    Returns a dictionary to be used with :meth:`aerospike.Client.operate` and :meth:`aerospike.Client.operate_ordered`.

    Args:
        bin_name (str): The name of the bin to be operated on.
        index_bit_count: An optional number of index bits. Must be bewtween 4 and 16 inclusive.
        mh_bit_count: An optional number of min hash bits. Must be bewtween 4 and 58 inclusive.
        policy (dict): An optional dictionary of :ref:`HyperLogLog policies <aerospike_hll_policies>`.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_INIT,
        BIN_KEY: bin_name,
        INDEX_BIT_COUNT_KEY: -1 if index_bit_count is None else index_bit_count,
        MH_BIT_COUNT_KEY: -1 if mh_bit_count is None else mh_bit_count
    }

    if policy:
        op_dict[HLL_POLICY_KEY] = policy

    return op_dict

def hll_refresh_count(bin_name: str):
    """Creates a hll_refresh_count operation.

    Server updates the cached count if it is stale.
    Server returns the count.

    Returns a dictionary to be used with :meth:`aerospike.Client.operate` and :meth:`aerospike.Client.operate_ordered`.
    Args:
        bin_name (str): The name of the bin to be operated on.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_REFRESH_COUNT,
        BIN_KEY: bin_name,
    }

    return op_dict

def hll_set_union(bin_name: str, hll_list, policy=None):
    """Creates a hll_set_union operation.

    Server sets the union of all specified HLL objects with the HLL bin.
    Server returns nothing.

    Returns a dictionary to be used with :meth:`aerospike.Client.operate` and :meth:`aerospike.Client.operate_ordered`.

    Args:
        bin_name (str): The name of the bin to be operated on.
        hll_list (list): The HLLs who's union will be set.
        policy (dict): An optional dictionary of :ref:`HyperLogLog policies <aerospike_hll_policies>`.
    """
    op_dict = {
        OP_KEY: aerospike.OP_HLL_SET_UNION,
        BIN_KEY: bin_name,
        VALUE_LIST_KEY: hll_list
    }

    if policy:
        op_dict[HLL_POLICY_KEY] = policy

    return op_dict
<|endoftext|>


<commit_before># Software License Agreement (Apache License)
#
# Copyright (c) 2015, TU Delft Robotics Institute
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# author: G.A. vd. Hoorn (TU Delft Robotics Institute)
#

import math
import os
import logging

from ..mini_urdf import *
from ..mini_xacro import *

from frfformats.frw import FrwShapeKind

class UnknownFanucFrwObjectKind(Exception):
    pass

_ZERO_ORIGIN=[0, 0, 0, 0, 0, 0]

def convert_frw_box(box):
    """
    :param box: The frw box geometry to convert to its urdf counter part, ``FrwCellObject``
    :returns: urdf element sequence for a box geometry, ``str``
    """
    return gen_urdf_box(size=[box.scale.x, box.scale.y, box.scale.z])

def convert_frw_sphere(sphere):
    """
    :param sphere: The frw sphere geometry to convert to its urdf counter part, ``FrwCellObject``
    :returns: urdf element sequence for a sphere geometry, ``str``
    """
    return gen_urdf_sphere(radius=sphere.scale.x)

def convert_frw_cylinder(cylinder):
    """
    :param cylinder: The frw cylinder geometry to convert to its urdf counter part, ``FrwCellObject``
    :returns: urdf element sequence for a cylinder geometry, ``str``
    """
    # scale.x is divided by two as Roboguide stores cylinder radius
    return gen_urdf_cylinder(length=cylinder.scale.y, radius=cylinder.scale.x/2.0)

def convert_frw_cad_obj(filename):
    """
    :param filename: Path to CAD file used in frw for a CAD geometry object, ``str``
    :returns: urdf element sequence for a mesh geometry, ``str``
    """
    return gen_urdf_mesh(filename=filename)

def convert_frw_triad(triad):
    """
    :param triad: Instance of a frw triad to convert into a ROS pose, ``FrwTriad``
    :returns: A ROS compatible pose (6 floats: xyz (meter), rpy (radians)) that is a direct conversion of the frw triad, ``seq(float)``
    """
    return [
        # convert position to meters first
        triad.x / 1000.0, triad.y / 1000.0, triad.z / 1000.0,
        # TODO: check order of rotation axes
        math.radians(triad.w), math.radians(triad.p), math.radians(triad.r)]

def convert_frw_cell_obj(obj, links, joints, pkg, name_prefix):
    origin = convert_frw_triad(obj.location)
    geom_origin = _ZERO_ORIGIN[:]
    # bare link name, xacro prefixing is done later
    link_name = 'link_{0}_{1}'.format(name_prefix, obj.name)

    if obj.kind == FrwShapeKind.BOX:
        geom = convert_frw_box(obj)
        # origin of box in Roboguide is on top plane. ROS has origin
        # at box centre. We want frame location to be identical to Roboguide,
        # as well as appearance. To achieve that, we offset the geometry by
        # half the geometry height.
        # TODO: do this in a nicer way
        geom_origin[2] -= (obj.scale.z / 2.0)

    elif obj.kind == FrwShapeKind.SPHERE:
        geom = convert_frw_sphere(obj)

    elif obj.kind == FrwShapeKind.CYLINDER:
        geom = convert_frw_cylinder(obj)
        # the geometry of cylinders in Roboguide is rotated PI/-2 over the
        # X-axis compared to ROS. To keep joint frame pose identical, we
        # compensate the geometry origin by rotating that by PI/-2.
        geom_origin[3] = math.pi/-2.0

    elif obj.kind == FrwShapeKind.CAD:
        mesh_file = os.path.basename(obj.cad_file_cached.replace('\\', '/'))
        mesh_file = '{0}.stl'.format(os.path.splitext(mesh_file)[0])
        geom = convert_frw_cad_obj(filename='package://{0}/{1}'.format(pkg, mesh_file))

    else:
        raise UnknownFanucFrwObjectKind("Cannot convert '{0}' as '{1}' is "
            "unsupported".format(obj.name, obj.kind.name))

    link = gen_link(
        name='${prefix}' + link_name,
        visual=gen_urdf_visual(
            geom=geom,
            material=gen_urdf_material(obj.colour.as_rgba_tuple()),
            origin=gen_urdf_origin(geom_origin))
    )

    joint = gen_joint_fixed(
        name='${prefix}base_link-' + link_name,
        parent='${prefix}base_link',
        child='${prefix}' + link_name,
        origin=gen_urdf_origin(origin)
    )

    links.append(link)
    joints.append(joint)

def convert_frw_cell_objs(objs, links, joints, pkg, prefix):
    for obj in objs:
        try:
            convert_frw_cell_obj(obj, links, joints, pkg, prefix)
        except Exception, e:
            logging.warning("Skipping '%s', error processing: %s" % (obj.name, e))

def convert_frw_uframes(ctrlrs, links, joints):
    for ctrlr in ctrlrs:
        for grp in ctrlr.robot_groups:

            # add frame for motion group. Note: this should coincide with
            # pose of robot itself
            grp_link_name = 'c{cidx}_g{gid}_{name}'.format(
                cidx=ctrlr.id,
                gid=grp.id,
                name=grp.name)
            grp_frame_origin = convert_frw_triad(grp.location)

            links.append(gen_link(name="${prefix}" + grp_link_name))
            joints.append(gen_joint_fixed(
                name='${prefix}base_link-' + grp_link_name,
                parent='${prefix}base_link',
                child='${prefix}' + grp_link_name,
                origin=gen_urdf_origin(grp_frame_origin)
            ))

            for uframe in grp.user_frames:
                # TODO: for now we use 'all zeros' to check for undefined user frames
                if uframe.location.all_zeros():
                    continue

                uframe_name = 'c{cidx}_g{gid}_u{uid}_{name}'.format(
                    cidx=ctrlr.id,
                    gid=grp.id,
                    uid=uframe.num,
                    name=uframe.name)
                uframe_origin = convert_frw_triad(uframe.location)
                # TODO: user frames are relative to robot world frame, which we
                # don't have access to at this time. Figure out some way to
                # fix this.

                # empty link
                links.append(gen_link(name="${prefix}" + uframe_name))
                joints.append(gen_joint_fixed(
                    name='${prefix}base_link-' + uframe_name,
                    parent='${prefix}' + grp_link_name,
                    child='${prefix}' + uframe_name,
                    origin=gen_urdf_origin(uframe_origin)
                ))

def convert_frw(frw_cell, macro_name, pkg, include_uframes=False):
    links = []
    joints = []

    # cell base link
    links.append(gen_link(name="${prefix}base_link"))

    # convert all supported cell objects
    convert_frw_cell_objs(frw_cell.obstacles, links, joints, pkg, 'obstacle')
    convert_frw_cell_objs(frw_cell.fixtures, links, joints, pkg, 'fixture')

    if include_uframes:
        convert_frw_uframes(frw_cell.robot_controllers, links, joints)

    # generate rest of file and return
    return gen_xacro(
        gen_xacro_robot(
            gen_xacro_macro(macro_name, joints=joints, links=links)))
<commit_msg>frw2xacro: sync with FrwCellObject change in frfformats.<commit_after># Software License Agreement (Apache License)
#
# Copyright (c) 2015, TU Delft Robotics Institute
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# author: G.A. vd. Hoorn (TU Delft Robotics Institute)
#

import math
import os
import logging

from ..mini_urdf import *
from ..mini_xacro import *

from frfformats.frw import FrwShapeKind

class UnknownFanucFrwObjectKind(Exception):
    pass

_ZERO_ORIGIN=[0, 0, 0, 0, 0, 0]

def convert_frw_box(box):
    """
    :param box: The frw box geometry to convert to its urdf counter part, ``FrwCellObject``
    :returns: urdf element sequence for a box geometry, ``str``
    """
    return gen_urdf_box(size=[box.scale.x, box.scale.y, box.scale.z])

def convert_frw_sphere(sphere):
    """
    :param sphere: The frw sphere geometry to convert to its urdf counter part, ``FrwCellObject``
    :returns: urdf element sequence for a sphere geometry, ``str``
    """
    return gen_urdf_sphere(radius=sphere.scale.x)

def convert_frw_cylinder(cylinder):
    """
    :param cylinder: The frw cylinder geometry to convert to its urdf counter part, ``FrwCellObject``
    :returns: urdf element sequence for a cylinder geometry, ``str``
    """
    # scale.x is divided by two as Roboguide stores cylinder radius
    return gen_urdf_cylinder(length=cylinder.scale.y, radius=cylinder.scale.x/2.0)

def convert_frw_cad_obj(filename):
    """
    :param filename: Path to CAD file used in frw for a CAD geometry object, ``str``
    :returns: urdf element sequence for a mesh geometry, ``str``
    """
    return gen_urdf_mesh(filename=filename)

def convert_frw_triad(triad):
    """
    :param triad: Instance of a frw triad to convert into a ROS pose, ``FrwTriad``
    :returns: A ROS compatible pose (6 floats: xyz (meter), rpy (radians)) that is a direct conversion of the frw triad, ``seq(float)``
    """
    return [
        # convert position to meters first
        triad.x / 1000.0, triad.y / 1000.0, triad.z / 1000.0,
        # TODO: check order of rotation axes
        math.radians(triad.w), math.radians(triad.p), math.radians(triad.r)]

def convert_frw_cell_obj(obj, links, joints, pkg, name_prefix):
    origin = convert_frw_triad(obj.location)
    geom_origin = _ZERO_ORIGIN[:]
    # bare link name, xacro prefixing is done later
    link_name = 'link_{0}_{1}'.format(name_prefix, obj.name)

    if obj.kind == FrwShapeKind.BOX:
        geom = convert_frw_box(obj)
        # origin of box in Roboguide is on top plane. ROS has origin
        # at box centre. We want frame location to be identical to Roboguide,
        # as well as appearance. To achieve that, we offset the geometry by
        # half the geometry height.
        # TODO: do this in a nicer way
        geom_origin[2] -= (obj.scale.z / 2.0)

    elif obj.kind == FrwShapeKind.SPHERE:
        geom = convert_frw_sphere(obj)

    elif obj.kind == FrwShapeKind.CYLINDER:
        geom = convert_frw_cylinder(obj)
        # the geometry of cylinders in Roboguide is rotated PI/-2 over the
        # X-axis compared to ROS. To keep joint frame pose identical, we
        # compensate the geometry origin by rotating that by PI/-2.
        geom_origin[3] = math.pi/-2.0

    elif obj.kind == FrwShapeKind.CAD:
        mesh_file = os.path.basename(obj.cad_file.cached_file.replace('\\', '/'))
        mesh_file = '{0}.stl'.format(os.path.splitext(mesh_file)[0])
        geom = convert_frw_cad_obj(filename='package://{0}/{1}'.format(pkg, mesh_file))

    else:
        raise UnknownFanucFrwObjectKind("Cannot convert '{0}' as '{1}' is "
            "unsupported".format(obj.name, obj.kind.name))

    link = gen_link(
        name='${prefix}' + link_name,
        visual=gen_urdf_visual(
            geom=geom,
            material=gen_urdf_material(obj.colour.as_rgba_tuple()),
            origin=gen_urdf_origin(geom_origin))
    )

    joint = gen_joint_fixed(
        name='${prefix}base_link-' + link_name,
        parent='${prefix}base_link',
        child='${prefix}' + link_name,
        origin=gen_urdf_origin(origin)
    )

    links.append(link)
    joints.append(joint)

def convert_frw_cell_objs(objs, links, joints, pkg, prefix):
    for obj in objs:
        try:
            convert_frw_cell_obj(obj, links, joints, pkg, prefix)
        except Exception, e:
            logging.warning("Skipping '%s', error processing: %s" % (obj.name, e))

def convert_frw_uframes(ctrlrs, links, joints):
    for ctrlr in ctrlrs:
        for grp in ctrlr.robot_groups:

            # add frame for motion group. Note: this should coincide with
            # pose of robot itself
            grp_link_name = 'c{cidx}_g{gid}_{name}'.format(
                cidx=ctrlr.id,
                gid=grp.id,
                name=grp.name)
            grp_frame_origin = convert_frw_triad(grp.location)

            links.append(gen_link(name="${prefix}" + grp_link_name))
            joints.append(gen_joint_fixed(
                name='${prefix}base_link-' + grp_link_name,
                parent='${prefix}base_link',
                child='${prefix}' + grp_link_name,
                origin=gen_urdf_origin(grp_frame_origin)
            ))

            for uframe in grp.user_frames:
                # TODO: for now we use 'all zeros' to check for undefined user frames
                if uframe.location.all_zeros():
                    continue

                uframe_name = 'c{cidx}_g{gid}_u{uid}_{name}'.format(
                    cidx=ctrlr.id,
                    gid=grp.id,
                    uid=uframe.num,
                    name=uframe.name)
                uframe_origin = convert_frw_triad(uframe.location)
                # TODO: user frames are relative to robot world frame, which we
                # don't have access to at this time. Figure out some way to
                # fix this.

                # empty link
                links.append(gen_link(name="${prefix}" + uframe_name))
                joints.append(gen_joint_fixed(
                    name='${prefix}base_link-' + uframe_name,
                    parent='${prefix}' + grp_link_name,
                    child='${prefix}' + uframe_name,
                    origin=gen_urdf_origin(uframe_origin)
                ))

def convert_frw(frw_cell, macro_name, pkg, include_uframes=False):
    links = []
    joints = []

    # cell base link
    links.append(gen_link(name="${prefix}base_link"))

    # convert all supported cell objects
    convert_frw_cell_objs(frw_cell.obstacles, links, joints, pkg, 'obstacle')
    convert_frw_cell_objs(frw_cell.fixtures, links, joints, pkg, 'fixture')

    if include_uframes:
        convert_frw_uframes(frw_cell.robot_controllers, links, joints)

    # generate rest of file and return
    return gen_xacro(
        gen_xacro_robot(
            gen_xacro_macro(macro_name, joints=joints, links=links)))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import print_function
import sys
import time
import json
import yaml
from functools import reduce

from ..external import six

from ..deployment.autoscaling import Autoscaling
from ..deployment.autoscaling.metrics import _AutoscalingMetric
from ..deployment.resources import _Resource
from ..deployment.update.rules import _UpdateRule
from ..deployment import DeployedModel
from ..deployment.update._strategies import _UpdateStrategy, DirectUpdateStrategy, CanaryUpdateStrategy
from .._internal_utils import _utils
from .._tracking import experimentrun
from .._registry import RegisteredModelVersion

def merge_dicts(a, b):
    result = a.copy()
    result.update(b)
    return result

class Endpoint(object):
    """
    Object representing an endpoint for deployment.

    There should not be a need to instantiate this class directly; please use
    :meth:`Client.create_endpoint`.

    Attributes
    ----------
    id : int
        ID of this Endpoint.
    path : str
        Path of this Endpoint.

    """
    def __init__(self, conn, conf, workspace, id):
        self.workspace = workspace
        self._conn = conn
        self._conf = conf
        self.id = id

    def __repr__(self):
        status = self.get_status()
        data = Endpoint._get_json_by_id(self._conn, self.workspace, self.id)

        return '\n'.join((
            "path: {}".format(data['creator_request']['path']),
            "id: {}".format(self.id),
            "status: {}".format(status["status"]),
            "date created: {}".format(data["date_created"]),
            "date updated: {}".format(data["date_updated"]),
            "stage's date created: {}".format(status["date_created"]),
            "stage's date updated: {}".format(status["date_updated"]),
            "components: {}".format(json.dumps(status["components"], indent=4)),
        ))

    @property
    def path(self):
        return self._path(Endpoint._get_json_by_id(self._conn, self.workspace, self.id))

    def _path(self, data):
        return data['creator_request']['path']

    def _date_updated(self, data):
        return data['date_updated']

    @classmethod
    def _create(cls, conn, conf, workspace, path, description=None):
        endpoint_json = cls._create_json(conn, workspace, path, description)
        if endpoint_json:
            return cls(conn, conf, workspace, endpoint_json['id'])
        else:
            return None

    @classmethod
    def _create_json(cls, conn, workspace, path, description=None):
        data = {}
        if description:
            data["description"] = description
        if not path.startswith('/'):
            path = '/' + path
        data["path"] = path
        url = "{}://{}/api/v1/deployment/workspace/{}/endpoints".format(conn.scheme, conn.socket, workspace)
        response = _utils.make_request("POST", url, conn, json=data)
        _utils.raise_for_http_error(response)
        return response.json()

    @classmethod
    def _get_by_id(cls, conn, conf, workspace, id):
        endpoint_json = cls._get_json_by_id(conn, workspace, id)
        if endpoint_json:
            return cls(conn, conf, workspace, endpoint_json['id'])
        else:
            return None

    def _get_info_list_by_id(self):
        data = Endpoint._get_json_by_id(self._conn, self.workspace, self.id)
        return [self._path(data), str(self.id), self._date_updated(data)]

    @classmethod
    def _get_json_by_id(cls, conn, workspace, id):
        url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}".format(conn.scheme, conn.socket, workspace, id)
        response = _utils.make_request("GET", url, conn)
        _utils.raise_for_http_error(response)
        return response.json()

    @classmethod
    def _get_or_create_by_name(cls, conn, name, getter, creator):
        obj = getter(name)
        if obj is None:
            obj = creator(name)
        else:
            print("got existing {}: {}".format(cls.__name__, name))
        return obj

    @classmethod
    def _get_by_path(cls, conn, conf, workspace, path):
        endpoint_json = cls._get_json_by_path(conn, workspace, path)
        if endpoint_json:
            return cls(conn, conf, workspace, endpoint_json['id'])
        else:
            return None

    @classmethod
    def _get_endpoints(cls, conn, workspace):
        url = "{}://{}/api/v1/deployment/workspace/{}/endpoints".format(conn.scheme, conn.socket, workspace)
        response = _utils.make_request("GET", url, conn)
        _utils.raise_for_http_error(response)
        data_response = response.json()
        return data_response['endpoints']

    @classmethod
    def _get_json_by_path(cls, conn, workspace, path):
        endpoints = cls._get_endpoints(conn, workspace)
        if not path.startswith('/'):
            path = '/' + path
        for endpoint in endpoints:
            creator_request = endpoint['creator_request']
            if creator_request['path'] == path:
                return endpoint
        return None

    def update(self, model_reference, strategy, wait=False, resources=None, autoscaling=None, env_vars=None):
        """
        Updates the Endpoint with a model logged in an Experiment Run or a Model Version.

        Parameters
        ----------
        model_reference : :class:`~verta._tracking.experimentrun.ExperimentRun` or :class:`~verta._registry.modelversion.RegisteredModelVersion`
            An Experiment Run or a Model Version with a model logged.
        strategy : :class:`~verta.deployment.update._strategies._UpdateStrategy`
            Strategy (direct or canary) for updating the Endpoint.
        wait : bool
            Whether to wait for the Endpoint to finish updating before returning.
        resources : list of :class:`~verta.deployment.resources._Resource`
            Resources allowed for the updated Endpoint.
        autoscaling : :class:`~verta.deployment.autoscaling._autoscaling.Autoscaling`
            Autoscaling condition for the updated Endpoint.
        env_vars : dict of str to str
            Environment variables.

        Returns
        -------
        status : dict of str to {None, bool, float, int, str, list, dict}

        """
        if not isinstance(strategy, _UpdateStrategy):
            raise TypeError("`strategy` must be an object from verta.deployment.strategies")

        if not isinstance(model_reference, (RegisteredModelVersion, experimentrun.ExperimentRun)):
            raise TypeError("`model_reference` must be an ExperimentRun or RegisteredModelVersion")

        if autoscaling and not isinstance(autoscaling, Autoscaling):
            raise TypeError("`autoscaling` must be an Autoscaling object")

        if env_vars:
            env_vars_err_msg = "`env_vars` must be dictionary of str keys and str values"
            if not isinstance(env_vars, dict):
                raise TypeError(env_vars_err_msg)
            for key, value in env_vars.items():
                if not isinstance(key, six.string_types) or not isinstance(value, six.string_types):
                    raise TypeError(env_vars_err_msg)

        # Create new build:
        build_id = self._create_build(model_reference)
        return self._update_from_build(build_id, strategy, wait, resources, autoscaling, env_vars)

    def _update_from_build(self, build_id, strategy, wait=False, resources=None, autoscaling=None, env_vars=None):
        update_body = self._form_update_body(resources, strategy, autoscaling, env_vars, build_id)

        # Update stages with new build
        url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages/{}/update".format(
            self._conn.scheme,
            self._conn.socket,
            self.workspace,
            self.id,
            self._get_or_create_stage(),
        )
        response = _utils.make_request("PUT", url, self._conn, json=update_body)
        _utils.raise_for_http_error(response)

        if wait:
            print("waiting for update...", end='')
            sys.stdout.flush()

            status_dict = self.get_status()  # if you think of a better var name, please do use it haha
            while status_dict['status'] not in ("active", "error") \
                    or (status_dict['status'] == "active" and len(self.get_status()['components']) > 1):
                print(".", end='')
                sys.stdout.flush()
                time.sleep(5)
                status_dict = self.get_status()

            print()
            if self.get_status()['status'] == "error":
                raise RuntimeError("endpoint update failed")

        return self.get_status()

    def _create_build(self, model_reference):
        url = "{}://{}/api/v1/deployment/workspace/{}/builds".format(
            self._conn.scheme,
            self._conn.socket,
            self.workspace,
        )

        if isinstance(model_reference, RegisteredModelVersion):
            response = _utils.make_request("POST", url, self._conn, json={"model_version_id": model_reference.id})
        elif isinstance(model_reference, experimentrun.ExperimentRun):
            response = _utils.make_request("POST", url, self._conn, json={"run_id": model_reference.id})
        else:
            raise TypeError("`model_reference` must be an ExperimentRun or RegisteredModelVersion")

        _utils.raise_for_http_error(response)
        return response.json()["id"]

    def _get_or_create_stage(self, name="production"):
        if name == "production":
            # Check if a stage exists:
            url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages".format(
                self._conn.scheme,
                self._conn.socket,
                self.workspace,
                self.id
            )
            response = _utils.make_request("GET", url, self._conn, params={})

            _utils.raise_for_http_error(response)
            response_json = response.json()

            if response_json["stages"]:
                return response_json["stages"][0]["id"]

            # no stage found:
            return self._create_stage("production")
        else:
            raise NotImplementedError("currently not supported other stages")

    def _create_stage(self, name="production"):
        url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages".format(
            self._conn.scheme,
            self._conn.socket,
            self.workspace,
            self.id
        )
        response = _utils.make_request("POST", url, self._conn, json={"name": name})
        _utils.raise_for_http_error(response)
        return response.json()["id"]

    def update_from_config(self, filepath):
        """
        Updates the Endpoint via a YAML or JSON config file.

        Parameters
        ----------
        filepath : str
            Path to the YAML or JSON config file.

        Returns
        -------
        status : dict of str to {None, bool, float, int, str, list, dict}

        Raises
        ------
        ValueError
            If the file is not JSON or YAML.

        """
        update_dict = None

        with open(filepath, 'r') as f:
            config = f.read()

        try:
            update_dict = json.loads(config)
        except ValueError:
            pass

        if not update_dict:
            try:
                update_dict = yaml.safe_load(config)
            except yaml.YAMLError:
                pass

        if not update_dict:
            raise ValueError("input file must be a json or yaml")

        return self._update_from_dict(update_dict)

    def _update_from_dict(self, update_dict):
        if update_dict["strategy"] == "direct":
            strategy = DirectUpdateStrategy()
        elif update_dict["strategy"] == "canary":
            strategy = CanaryUpdateStrategy(
                interval=int(update_dict["canary_strategy"]["progress_interval_seconds"]),
                step=float(update_dict["canary_strategy"]["progress_step"])
            )

            for rule in update_dict["canary_strategy"]["rules"]:
                strategy.add_rule(_UpdateRule._from_dict(rule))
        else:
            raise ValueError("update strategy must be \"direct\" or \"canary\"")

        if "autoscaling" in update_dict:
            autoscaling_obj = Autoscaling._from_dict(update_dict["autoscaling"]["quantities"])

            for metric in update_dict["autoscaling"]["metrics"]:
                autoscaling_obj.add_metric(_AutoscalingMetric._from_dict(metric))
        else:
            autoscaling_obj = None

        if "resources" in update_dict:
            resources_list = _Resource._from_dict(update_dict["resources"])
        else:
            resources_list = None

        if "run_id" in update_dict and "model_version_id" in update_dict:
            raise ValueError("cannot provide both run_id and model_version_id")
        elif "run_id" in update_dict:
            model_reference = experimentrun.ExperimentRun._get_by_id(self._conn, self._conf, id=update_dict["run_id"])
        elif "model_version_id" in update_dict:
            model_reference = RegisteredModelVersion._get_by_id(self._conn, self._conf, id=update_dict["model_version_id"])
        else:
            raise RuntimeError("must provide either model_version_id or run_id")

        return self.update(model_reference, strategy, resources=resources_list, autoscaling=autoscaling_obj, env_vars=update_dict.get("env_vars"))

    def get_status(self):
        """
        Gets status on the endpoint.

        Returns
        -------
        status : dict of str to {None, bool, float, int, str, list, dict}

        """
        url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages/{}".format(
            self._conn.scheme,
            self._conn.socket,
            self.workspace,
            self.id,
            self._get_or_create_stage()
        )
        response = _utils.make_request("GET", url, self._conn)
        _utils.raise_for_http_error(response)
        response_json = response.json()
        response_json["stage_id"] = response_json.pop("id")

        return response_json

    def get_access_token(self):
        """
        Gets the access token of the Endpoint.

        Returns
        -------
        str or None

        """
        url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages/{}/accesstokens".format(
            self._conn.scheme,
            self._conn.socket,
            self.workspace,
            self.id,
            self._get_or_create_stage(),
        )
        response = _utils.make_request("GET", url, self._conn)
        _utils.raise_for_http_error(response)
        data = response.json()
        tokens = data["tokens"]
        if len(tokens) == 0:
            return None
        return tokens[0]['creator_request']['value']

    def _form_update_body(self, resources, strategy, autoscaling, env_vars, build_id):
        update_body = strategy._as_build_update_req_body(build_id)
        if resources:
            update_body["resources"] = reduce(lambda resource_a, resource_b: merge_dicts(resource_a, resource_b),
                                              map(lambda resource: resource.to_dict(), resources))

        if autoscaling:
            update_body["autoscaling"] = autoscaling._as_dict()

        if env_vars:
            update_body["env"] = list(map(lambda env_var: {"name": env_var, "value": env_vars[env_var]}, env_vars))

        # prepare body for update request
        return update_body

    def get_deployed_model(self):
        """
        Returns an object for making predictions against the deployed model.

        Returns
        -------
        :class:`~verta.deployment.DeployedModel`

        Raises
        ------
        RuntimeError
            If the model is not currently deployed.

        """
        status = self.get_status()
        if status['status'] not in ("active", "updating"):
            raise RuntimeError("model is not currently deployed (status: {})".format(status))

        access_token = self.get_access_token()
        url = "{}://{}/api/v1/predict{}".format(self._conn.scheme, self._conn.socket, self.path)
        return DeployedModel.from_url(url, access_token)

    def get_update_status(self):
        """
        Gets update status on the endpoint.

        Returns
        -------
        update_status : dict of str to {None, bool, float, int, str, list, dict}

        """
        url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages/{}/status".format(
            self._conn.scheme,
            self._conn.socket,
            self.workspace,
            self.id,
            self._get_or_create_stage()
        )
        response = _utils.make_request("GET", url, self._conn)
        _utils.raise_for_http_error(response)
        return response.json()
<commit_msg>Display build errors for endpoint.update(wait=True) (#1290)<commit_after># -*- coding: utf-8 -*-

from __future__ import print_function
import sys
import time
import json
import yaml
from functools import reduce

from ..external import six

from ..deployment.autoscaling import Autoscaling
from ..deployment.autoscaling.metrics import _AutoscalingMetric
from ..deployment.resources import _Resource
from ..deployment.update.rules import _UpdateRule
from ..deployment import DeployedModel
from ..deployment.update._strategies import _UpdateStrategy, DirectUpdateStrategy, CanaryUpdateStrategy
from .._internal_utils import _utils
from .._tracking import experimentrun
from .._registry import RegisteredModelVersion

def merge_dicts(a, b):
    result = a.copy()
    result.update(b)
    return result

class Endpoint(object):
    """
    Object representing an endpoint for deployment.

    There should not be a need to instantiate this class directly; please use
    :meth:`Client.create_endpoint`.

    Attributes
    ----------
    id : int
        ID of this Endpoint.
    path : str
        Path of this Endpoint.

    """
    def __init__(self, conn, conf, workspace, id):
        self.workspace = workspace
        self._conn = conn
        self._conf = conf
        self.id = id

    def __repr__(self):
        status = self.get_status()
        data = Endpoint._get_json_by_id(self._conn, self.workspace, self.id)

        return '\n'.join((
            "path: {}".format(data['creator_request']['path']),
            "id: {}".format(self.id),
            "status: {}".format(status["status"]),
            "date created: {}".format(data["date_created"]),
            "date updated: {}".format(data["date_updated"]),
            "stage's date created: {}".format(status["date_created"]),
            "stage's date updated: {}".format(status["date_updated"]),
            "components: {}".format(json.dumps(status["components"], indent=4)),
        ))

    @property
    def path(self):
        return self._path(Endpoint._get_json_by_id(self._conn, self.workspace, self.id))

    def _path(self, data):
        return data['creator_request']['path']

    def _date_updated(self, data):
        return data['date_updated']

    @classmethod
    def _create(cls, conn, conf, workspace, path, description=None):
        endpoint_json = cls._create_json(conn, workspace, path, description)
        if endpoint_json:
            return cls(conn, conf, workspace, endpoint_json['id'])
        else:
            return None

    @classmethod
    def _create_json(cls, conn, workspace, path, description=None):
        data = {}
        if description:
            data["description"] = description
        if not path.startswith('/'):
            path = '/' + path
        data["path"] = path
        url = "{}://{}/api/v1/deployment/workspace/{}/endpoints".format(conn.scheme, conn.socket, workspace)
        response = _utils.make_request("POST", url, conn, json=data)
        _utils.raise_for_http_error(response)
        return response.json()

    @classmethod
    def _get_by_id(cls, conn, conf, workspace, id):
        endpoint_json = cls._get_json_by_id(conn, workspace, id)
        if endpoint_json:
            return cls(conn, conf, workspace, endpoint_json['id'])
        else:
            return None

    def _get_info_list_by_id(self):
        data = Endpoint._get_json_by_id(self._conn, self.workspace, self.id)
        return [self._path(data), str(self.id), self._date_updated(data)]

    @classmethod
    def _get_json_by_id(cls, conn, workspace, id):
        url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}".format(conn.scheme, conn.socket, workspace, id)
        response = _utils.make_request("GET", url, conn)
        _utils.raise_for_http_error(response)
        return response.json()

    @classmethod
    def _get_or_create_by_name(cls, conn, name, getter, creator):
        obj = getter(name)
        if obj is None:
            obj = creator(name)
        else:
            print("got existing {}: {}".format(cls.__name__, name))
        return obj

    @classmethod
    def _get_by_path(cls, conn, conf, workspace, path):
        endpoint_json = cls._get_json_by_path(conn, workspace, path)
        if endpoint_json:
            return cls(conn, conf, workspace, endpoint_json['id'])
        else:
            return None

    @classmethod
    def _get_endpoints(cls, conn, workspace):
        url = "{}://{}/api/v1/deployment/workspace/{}/endpoints".format(conn.scheme, conn.socket, workspace)
        response = _utils.make_request("GET", url, conn)
        _utils.raise_for_http_error(response)
        data_response = response.json()
        return data_response['endpoints']

    @classmethod
    def _get_json_by_path(cls, conn, workspace, path):
        endpoints = cls._get_endpoints(conn, workspace)
        if not path.startswith('/'):
            path = '/' + path
        for endpoint in endpoints:
            creator_request = endpoint['creator_request']
            if creator_request['path'] == path:
                return endpoint
        return None

    def update(self, model_reference, strategy, wait=False, resources=None, autoscaling=None, env_vars=None):
        """
        Updates the Endpoint with a model logged in an Experiment Run or a Model Version.

        Parameters
        ----------
        model_reference : :class:`~verta._tracking.experimentrun.ExperimentRun` or :class:`~verta._registry.modelversion.RegisteredModelVersion`
            An Experiment Run or a Model Version with a model logged.
        strategy : :class:`~verta.deployment.update._strategies._UpdateStrategy`
            Strategy (direct or canary) for updating the Endpoint.
        wait : bool
            Whether to wait for the Endpoint to finish updating before returning.
        resources : list of :class:`~verta.deployment.resources._Resource`
            Resources allowed for the updated Endpoint.
        autoscaling : :class:`~verta.deployment.autoscaling._autoscaling.Autoscaling`
            Autoscaling condition for the updated Endpoint.
        env_vars : dict of str to str
            Environment variables.

        Returns
        -------
        status : dict of str to {None, bool, float, int, str, list, dict}

        """
        if not isinstance(strategy, _UpdateStrategy):
            raise TypeError("`strategy` must be an object from verta.deployment.strategies")

        if not isinstance(model_reference, (RegisteredModelVersion, experimentrun.ExperimentRun)):
            raise TypeError("`model_reference` must be an ExperimentRun or RegisteredModelVersion")

        if autoscaling and not isinstance(autoscaling, Autoscaling):
            raise TypeError("`autoscaling` must be an Autoscaling object")

        if env_vars:
            env_vars_err_msg = "`env_vars` must be dictionary of str keys and str values"
            if not isinstance(env_vars, dict):
                raise TypeError(env_vars_err_msg)
            for key, value in env_vars.items():
                if not isinstance(key, six.string_types) or not isinstance(value, six.string_types):
                    raise TypeError(env_vars_err_msg)

        # Create new build:
        build_id = self._create_build(model_reference)
        return self._update_from_build(build_id, strategy, wait, resources, autoscaling, env_vars)

    def _update_from_build(self, build_id, strategy, wait=False, resources=None, autoscaling=None, env_vars=None):
        update_body = self._form_update_body(resources, strategy, autoscaling, env_vars, build_id)

        # Update stages with new build
        url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages/{}/update".format(
            self._conn.scheme,
            self._conn.socket,
            self.workspace,
            self.id,
            self._get_or_create_stage(),
        )
        response = _utils.make_request("PUT", url, self._conn, json=update_body)
        _utils.raise_for_http_error(response)

        if wait:
            print("waiting for update...", end='')
            sys.stdout.flush()

            status_dict = self.get_status()  # if you think of a better var name, please do use it haha
            while status_dict['status'] not in ("active", "error") \
                    or (status_dict['status'] == "active" and len(self.get_status()['components']) > 1):
                print(".", end='')
                sys.stdout.flush()
                time.sleep(5)
                status_dict = self.get_status()

            print()
            if self.get_status()['status'] == "error":
                failure_msg = self.get_status()['components'][0].get('message', "no error message available")
                raise RuntimeError("endpoint update failed;\n{}".format(failure_msg))

        return self.get_status()

    def _create_build(self, model_reference):
        url = "{}://{}/api/v1/deployment/workspace/{}/builds".format(
            self._conn.scheme,
            self._conn.socket,
            self.workspace,
        )

        if isinstance(model_reference, RegisteredModelVersion):
            response = _utils.make_request("POST", url, self._conn, json={"model_version_id": model_reference.id})
        elif isinstance(model_reference, experimentrun.ExperimentRun):
            response = _utils.make_request("POST", url, self._conn, json={"run_id": model_reference.id})
        else:
            raise TypeError("`model_reference` must be an ExperimentRun or RegisteredModelVersion")

        _utils.raise_for_http_error(response)
        return response.json()["id"]

    def _get_or_create_stage(self, name="production"):
        if name == "production":
            # Check if a stage exists:
            url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages".format(
                self._conn.scheme,
                self._conn.socket,
                self.workspace,
                self.id
            )
            response = _utils.make_request("GET", url, self._conn, params={})

            _utils.raise_for_http_error(response)
            response_json = response.json()

            if response_json["stages"]:
                return response_json["stages"][0]["id"]

            # no stage found:
            return self._create_stage("production")
        else:
            raise NotImplementedError("currently not supported other stages")

    def _create_stage(self, name="production"):
        url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages".format(
            self._conn.scheme,
            self._conn.socket,
            self.workspace,
            self.id
        )
        response = _utils.make_request("POST", url, self._conn, json={"name": name})
        _utils.raise_for_http_error(response)
        return response.json()["id"]

    def update_from_config(self, filepath):
        """
        Updates the Endpoint via a YAML or JSON config file.

        Parameters
        ----------
        filepath : str
            Path to the YAML or JSON config file.

        Returns
        -------
        status : dict of str to {None, bool, float, int, str, list, dict}

        Raises
        ------
        ValueError
            If the file is not JSON or YAML.

        """
        update_dict = None

        with open(filepath, 'r') as f:
            config = f.read()

        try:
            update_dict = json.loads(config)
        except ValueError:
            pass

        if not update_dict:
            try:
                update_dict = yaml.safe_load(config)
            except yaml.YAMLError:
                pass

        if not update_dict:
            raise ValueError("input file must be a json or yaml")

        return self._update_from_dict(update_dict)

    def _update_from_dict(self, update_dict):
        if update_dict["strategy"] == "direct":
            strategy = DirectUpdateStrategy()
        elif update_dict["strategy"] == "canary":
            strategy = CanaryUpdateStrategy(
                interval=int(update_dict["canary_strategy"]["progress_interval_seconds"]),
                step=float(update_dict["canary_strategy"]["progress_step"])
            )

            for rule in update_dict["canary_strategy"]["rules"]:
                strategy.add_rule(_UpdateRule._from_dict(rule))
        else:
            raise ValueError("update strategy must be \"direct\" or \"canary\"")

        if "autoscaling" in update_dict:
            autoscaling_obj = Autoscaling._from_dict(update_dict["autoscaling"]["quantities"])

            for metric in update_dict["autoscaling"]["metrics"]:
                autoscaling_obj.add_metric(_AutoscalingMetric._from_dict(metric))
        else:
            autoscaling_obj = None

        if "resources" in update_dict:
            resources_list = _Resource._from_dict(update_dict["resources"])
        else:
            resources_list = None

        if "run_id" in update_dict and "model_version_id" in update_dict:
            raise ValueError("cannot provide both run_id and model_version_id")
        elif "run_id" in update_dict:
            model_reference = experimentrun.ExperimentRun._get_by_id(self._conn, self._conf, id=update_dict["run_id"])
        elif "model_version_id" in update_dict:
            model_reference = RegisteredModelVersion._get_by_id(self._conn, self._conf, id=update_dict["model_version_id"])
        else:
            raise RuntimeError("must provide either model_version_id or run_id")

        return self.update(model_reference, strategy, resources=resources_list, autoscaling=autoscaling_obj, env_vars=update_dict.get("env_vars"))

    def get_status(self):
        """
        Gets status on the endpoint.

        Returns
        -------
        status : dict of str to {None, bool, float, int, str, list, dict}

        """
        url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages/{}".format(
            self._conn.scheme,
            self._conn.socket,
            self.workspace,
            self.id,
            self._get_or_create_stage()
        )
        response = _utils.make_request("GET", url, self._conn)
        _utils.raise_for_http_error(response)
        response_json = response.json()
        response_json["stage_id"] = response_json.pop("id")

        return response_json

    def get_access_token(self):
        """
        Gets the access token of the Endpoint.

        Returns
        -------
        str or None

        """
        url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages/{}/accesstokens".format(
            self._conn.scheme,
            self._conn.socket,
            self.workspace,
            self.id,
            self._get_or_create_stage(),
        )
        response = _utils.make_request("GET", url, self._conn)
        _utils.raise_for_http_error(response)
        data = response.json()
        tokens = data["tokens"]
        if len(tokens) == 0:
            return None
        return tokens[0]['creator_request']['value']

    def _form_update_body(self, resources, strategy, autoscaling, env_vars, build_id):
        update_body = strategy._as_build_update_req_body(build_id)
        if resources:
            update_body["resources"] = reduce(lambda resource_a, resource_b: merge_dicts(resource_a, resource_b),
                                              map(lambda resource: resource.to_dict(), resources))

        if autoscaling:
            update_body["autoscaling"] = autoscaling._as_dict()

        if env_vars:
            update_body["env"] = list(map(lambda env_var: {"name": env_var, "value": env_vars[env_var]}, env_vars))

        # prepare body for update request
        return update_body

    def get_deployed_model(self):
        """
        Returns an object for making predictions against the deployed model.

        Returns
        -------
        :class:`~verta.deployment.DeployedModel`

        Raises
        ------
        RuntimeError
            If the model is not currently deployed.

        """
        status = self.get_status()
        if status['status'] not in ("active", "updating"):
            raise RuntimeError("model is not currently deployed (status: {})".format(status))

        access_token = self.get_access_token()
        url = "{}://{}/api/v1/predict{}".format(self._conn.scheme, self._conn.socket, self.path)
        return DeployedModel.from_url(url, access_token)

    def get_update_status(self):
        """
        Gets update status on the endpoint.

        Returns
        -------
        update_status : dict of str to {None, bool, float, int, str, list, dict}

        """
        url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages/{}/status".format(
            self._conn.scheme,
            self._conn.socket,
            self.workspace,
            self.id,
            self._get_or_create_stage()
        )
        response = _utils.make_request("GET", url, self._conn)
        _utils.raise_for_http_error(response)
        return response.json()
<|endoftext|>


<commit_before># template_timestamps = [(t, template_id), ...]
def window(template_timestamps, window_size=60):
    isFirst = True
    windows = []
    current_window = set()

    for timestamped_template in timestamped_templates:
        t, template = timestamped_template
        currentTime = math.floor(int(float(t)) /  int(window_size))
        if isFirst:
            oldTime = currentTime
            isFirst = False
        if currentTime != oldTime:
            windows.append(current_window)
            current_window = set()
        current_window.add(timestamped_template)
        currentTime = oldTime

    if current_window:
        windows.append(current_window)

    return windows
<commit_msg>Changed window to use defaultdict to populate windows.<commit_after>import math
from collections import defaultdict

# timed_template = [(t, template_id), ...]
def window(timed_templates, window_size=60):
    windows = defaultdict(set)
    for timed_template in timed_templates:
        t, template = timed_template
        key = math.floor(int(float(t)) / int(window_size))
        windows[key].add(timed_template)
    return windows.values()

    """
    isFirst = True
    windows = []
    current_window = set()

    for timed_template in timed_templates:
        t, template = timed_template
        currentTime = math.floor(int(float(t)) / int(window_size))
        if isFirst:
            oldTime = currentTime
            isFirst = False
        if currentTime != oldTime:
            windows.append(current_window)
            current_window = set()
        current_window.add(timed_template)
        currentTime = oldTime

    if current_window:
        windows.append(current_window)

    return windows
    """
<|endoftext|>


<commit_before>from flask import Flask
from flask import current_app, request, abort

import hashlib
import json

class ApiSequrity(object):
    def __init__(self, secret):
        super(ApiSequrity, self).__init__()
        self.secret = secret

    def gen_key(self, uid: str):
        combined = self.secret + uid[:8]
        hash_str = hashlib.md5(combined.encode('utf-8')).hexdigest()
        return hash_str

    def check_key(self, uid: str, key: str):
        return self.gen_key(uid) == key

    def parse_header(self,request):
        uid,key = request.headers.get('Credentials', '').split('/')
        return uid,key

    def check_request_args(self, request):
        with current_app.app_context():
            debug = 'debug' in request.args
            key   = request.args.get('key')
            uid   = request.args.get('uid')
            if uid is None and key is None:
                uid,key = self.parse_header(request)
            if 'pass' not in request.args:
                if uid is None and key is None:
                    abort(400)
                if not self.check_key(uid, key):
                    abort(401, json.dumps({'message': uid + ' ' + key})) if debug else abort(401)
            return key, uid, debug
<commit_msg>no message<commit_after>from flask import Flask
from flask import current_app, request, abort

import hashlib
import json

class ApiSequrity(object):
    def __init__(self, secret):
        super(ApiSequrity, self).__init__()
        self.secret = secret

    def gen_key(self, uid: str):
        combined = self.secret + uid[:8]
        hash_str = hashlib.md5(combined.encode('utf-8')).hexdigest()
        return hash_str

    def check_key(self, uid: str, key: str):
        return self.gen_key(uid) == key

    def parse_header(self,request):
        uid,key = request.headers.get('Credentials', '').split('/')
        return uid,key

    def check_request_args(self, request):
        debug   = 'debug' in request.args
        canpass = current_app.config.get('DEBUG', False)
        key     = request.args.get('key')
        uid     = request.args.get('uid')
        with current_app.app_context():
            if uid is None and key is None:
                uid, key = self.parse_header(request)
            if not canpass or 'pass' not in request.args:
                if uid is None and key is None:
                    abort(400)
                if not self.check_key(uid, key):
                    abort(401, json.dumps({'message': uid + ' ' + key})) if debug else abort(401)
        return key, uid, debug
<|endoftext|>