<commit_before>from functools import lru_cache
from math import factorial
from typing import Callable

import jax
import jax.numpy as jnp

def sus(x):
    r"""Smooth Unit Step function.

    ``-inf->0, 0->0, 2->0.6, +inf->1``

    .. math::
        \text{sus}(x) = \begin{cases}
            0, & \text{if } x < 0 \\
            exp(-1/x), & \text{if } x \geq 0 \\
        \end{cases}

    """
    return jnp.where(x > 0.0, jnp.exp(-1.0 / jnp.where(x > 0.0, x, 1.0)), 0.0)

def soft_envelope(
    x: jnp.ndarray,
    x_max: float = 1.0,
    arg_multiplicator: float = 2.0,
    value_at_origin: float = 1.2,
) -> jnp.ndarray:
    r"""Smooth envelope function.

    .. jupyter-execute::
        :hide-code:

        import jax.numpy as jnp
        import e3nn_jax as e3nn
        import matplotlib.pyplot as plt

    .. jupyter-execute::

        x = jnp.linspace(0.0, 1.0, 100)
        plt.plot(x, e3nn.soft_envelope(x))

    Args:
        x (jnp.ndarray): input of shape ``[...]``
        x_max (float): cutoff value

    Returns:
        jnp.ndarray: smooth (:math:`C^\infty`) envelope function of shape ``[...]``
    """
    with jax.ensure_compile_time_eval():
        cste = value_at_origin / sus(arg_multiplicator)
    return cste * sus(arg_multiplicator * (1.0 - x / x_max))

def soft_one_hot_linspace(
    input: jnp.ndarray,
    *,
    start: float,
    end: float,
    number: int,
    basis: str = None,
    cutoff: bool = None,
    start_zero: bool = None,
    end_zero: bool = None,
):
    r"""Projection on a basis of functions.

    Returns a set of :math:`\{y_i(x)\}_{i=1}^N`,

    .. math::

        y_i(x) = \frac{1}{Z} f_i(x)

    where :math:`x` is the input and :math:`f_i` is the ith basis function.
    :math:`Z` is a constant defined such that,

    .. math::

        \langle \sum_{i=1}^N y_i(x)^2 \rangle_x \approx 1

    See the last plot below.

    Args:
        input (jnp.ndarray): input of shape ``[...]``
        start (float): minimum value span by the basis
        end (float): maximum value span by the basis
        number (int): number of basis functions :math:`N`
        basis (str): type of basis functions, one of ``gaussian``, ``cosine``, ``smooth_finite``, ``fourier``
        cutoff (bool): if ``True``, the basis functions are cutoff at the start and end of the interval
        start_zero (bool): if ``True``, the first basis function is zero at the start of the interval
        end_zero (bool): if ``True``, the last basis function is zero at the end of the interval

    Returns:
        jnp.ndarray: basis functions of shape ``[..., number]``

    Examples:

        .. jupyter-execute::
            :hide-code:

            import jax.numpy as jnp
            import numpy as np
            import e3nn_jax as e3nn
            import matplotlib.pyplot as plt

        .. jupyter-execute::

            bases = ["gaussian", "cosine", "smooth_finite", "fourier"]
            x = np.linspace(-1.0, 2.0, 200)

        .. jupyter-execute::

            fig, axss = plt.subplots(len(bases), 2, figsize=(9, 6), sharex=True, sharey=True)

            for axs, b in zip(axss, bases):
                for ax, c in zip(axs, [True, False]):
                    y = e3nn.soft_one_hot_linspace(x, start=-0.5, end=1.5, number=4, basis=b, cutoff=c)

                    plt.sca(ax)
                    plt.plot(x, y)
                    plt.plot([-0.5]*2, [-2, 2], "k-.")
                    plt.plot([1.5]*2, [-2, 2], "k-.")
                    plt.title(f"{b}" + (" with cutoff" if c else ""))

            plt.ylim(-1, 1.5)
            plt.tight_layout()

        .. jupyter-execute::

            fig, axss = plt.subplots(len(bases), 2, figsize=(9, 6), sharex=True, sharey=True)

            for axs, b in zip(axss, bases):
                for ax, c in zip(axs, [True, False]):
                    y = e3nn.soft_one_hot_linspace(x, start=-0.5, end=1.5, number=4, basis=b, cutoff=c)

                    plt.sca(ax)
                    plt.plot(x, np.sum(y**2, axis=-1))
                    plt.plot([-0.5]*2, [-2, 2], "k-.")
                    plt.plot([1.5]*2, [-2, 2], "k-.")
                    plt.title(f"{b}" + (" with cutoff" if c else ""))

            plt.ylim(0, 2)
            plt.tight_layout()
    """
    if cutoff is not None:
        assert start_zero is None
        assert end_zero is None
        start_zero = cutoff
        end_zero = cutoff

    del cutoff

    if start_zero not in [True, False]:
        raise ValueError("start_zero must be specified")

    if end_zero not in [True, False]:
        raise ValueError("end_zero must be specified")

    if start_zero and end_zero:
        values = jnp.linspace(start, end, number + 2)
        step = values[1] - values[0]
        values = values[1:-1]

    if start_zero and not end_zero:
        values = jnp.linspace(start, end, number + 1)
        step = values[1] - values[0]
        values = values[1:]

    if not start_zero and end_zero:
        values = jnp.linspace(start, end, number + 1)
        step = values[1] - values[0]
        values = values[:-1]

    if not start_zero and not end_zero:
        values = jnp.linspace(start, end, number)
        step = values[1] - values[0]

    diff = (input[..., None] - values) / step

    if basis == "gaussian":
        return jnp.exp(-(diff**2)) / 1.12

    if basis == "cosine":
        return jnp.where((-1.0 < diff) & (diff < 1.0), jnp.cos(jnp.pi / 2 * diff), 0.0)

    if basis == "smooth_finite":
        return 1.14136 * jnp.exp(2.0) * sus(diff + 1.0) * sus(1.0 - diff)

    if basis == "fourier":
        x = (input[..., None] - start) / (end - start)
        if start_zero and end_zero:
            i = jnp.arange(1, number + 1)
            return jnp.where((0.0 < x) & (x < 1.0), jnp.sin(jnp.pi * i * x) / jnp.sqrt(0.25 + number / 2), 0.0)
        elif not start_zero and not end_zero:
            i = jnp.arange(0, number)
            return jnp.cos(jnp.pi * i * x) / jnp.sqrt(0.25 + number / 2)
        else:
            raise ValueError("when using fourier basis, start_zero and end_zero must be the same")

    raise ValueError(f'basis="{basis}" is not a valid entry')

def bessel(x: jnp.ndarray, n: int, x_max: float = 1.0) -> jnp.ndarray:
    r"""Bessel basis functions.

    They obey the following normalization:

    .. math::

        \int_0^c r^2 B_n(r, c) B_m(r, c) dr = \delta_{nm}

    Args:
        x (jnp.ndarray): input of shape ``[...]``
        n (int): number of basis functions
        x_max (float): maximum value of the input

    Returns:
        jnp.ndarray: basis functions of shape ``[..., n]``

    Klicpera, J.; Groß, J.; Günnemann, S. Directional Message Passing for Molecular Graphs; ICLR 2020.
    Equation (7)
    """
    x = x[..., None]
    n = jnp.arange(1, n + 1)
    x_nonzero = jnp.where(x == 0.0, 1.0, x)
    return jnp.sqrt(2.0 / x_max) * jnp.where(
        x == 0,
        n * jnp.pi / x_max,
        jnp.sin(n * jnp.pi / x_max * x_nonzero) / x_nonzero,
    )

def u(p: int, x: jnp.ndarray) -> jnp.ndarray:
    r"""Equivalent to :func:`poly_envelope` with ``n0 = p-1`` and ``n1 = 2``."""
    return 1 - (p + 1) * (p + 2) / 2 * x**p + p * (p + 2) * x ** (p + 1) - p * (p + 1) / 2 * x ** (p + 2)

def _constraint(x: float, derivative: int, degree: int):
    return [0 if derivative > N else factorial(N) // factorial(N - derivative) * x ** (N - derivative) for N in range(degree)]

@lru_cache(maxsize=None)
def solve_polynomial(constraints) -> jnp.ndarray:
    with jax.ensure_compile_time_eval():
        jax_enable_x64 = jax.config.read("jax_enable_x64")
        jax.config.update("jax_enable_x64", True)

        degree = len(constraints)
        A = jnp.array(
            [_constraint(x, derivative, degree) for x, derivative, _ in sorted(constraints)],
            dtype=jnp.float64,
        )
        B = jnp.array([y for _, _, y in sorted(constraints)], dtype=jnp.float64)
        c = jnp.linalg.solve(A, B)

        jax.config.update("jax_enable_x64", jax_enable_x64)
    return jax.jit(lambda x: jnp.polyval(c[::-1], x))

def poly_envelope(n0: int, n1: int, x_max: float = 1.0) -> Callable[[float], float]:
    r"""Polynomial envelope function with ``n0`` and ``n1`` derivatives euqal to 0 at ``x=0`` and ``x=1`` respectively.

    Small documentation available at ``https://mariogeiger.ch/polynomial_envelope_for_gnn.pdf``.
    This is a generalization of :math:`u_p(x)`.

    .. jupyter-execute::
        :hide-code:

        import jax.numpy as jnp
        import e3nn_jax as e3nn
        import matplotlib.pyplot as plt

    .. jupyter-execute::

        x = jnp.linspace(0.0, 1.0, 100)
        plt.plot(x, e3nn.poly_envelope(10, 5)(x), label="10, 5")
        plt.plot(x, e3nn.poly_envelope(4, 4)(x), label="4, 4")
        plt.plot(x, e3nn.poly_envelope(1, 2)(x), label="1, 2")
        plt.legend()

    Args:
        n0 (int): number of derivatives equal to 0 at ``x=0``
        n1 (int): number of derivatives equal to 0 at ``x=1``
        x_max (float): maximum value of the input, instead of 1

    Returns:
        callable: polynomial envelope function
    """
    poly = solve_polynomial(
        frozenset(
            {(-0.5, 0, 1.0), (0.5, 0, 0.0)}
            | {(-0.5, derivative, 0.0) for derivative in range(1, n0 + 1)}
            | {(0.5, derivative, 0.0) for derivative in range(1, n1 + 1)}
        )
    )

    def f(x):
        x_small = jnp.where(x < x_max, x, x_max)
        return jnp.where(
            x < x_max,
            poly(x_small / x_max - 0.5),
            0.0,
        )

    return f
<commit_msg>edit docstring<commit_after>from functools import lru_cache
from math import factorial
from typing import Callable

import jax
import jax.numpy as jnp

def sus(x):
    r"""Smooth Unit Step function.

    ``-inf->0, 0->0, 2->0.6, +inf->1``

    .. math::
        \text{sus}(x) = \begin{cases}
            0, & \text{if } x < 0 \\
            exp(-1/x), & \text{if } x \geq 0 \\
        \end{cases}

    """
    return jnp.where(x > 0.0, jnp.exp(-1.0 / jnp.where(x > 0.0, x, 1.0)), 0.0)

def soft_envelope(
    x: jnp.ndarray,
    x_max: float = 1.0,
    arg_multiplicator: float = 2.0,
    value_at_origin: float = 1.2,
) -> jnp.ndarray:
    r"""Smooth envelope function.

    .. jupyter-execute::
        :hide-code:

        import jax.numpy as jnp
        import e3nn_jax as e3nn
        import matplotlib.pyplot as plt

    .. jupyter-execute::

        x = jnp.linspace(0.0, 1.0, 100)
        plt.plot(x, e3nn.soft_envelope(x))

    Args:
        x (jnp.ndarray): input of shape ``[...]``
        x_max (float): cutoff value

    Returns:
        jnp.ndarray: smooth (:math:`C^\infty`) envelope function of shape ``[...]``
    """
    with jax.ensure_compile_time_eval():
        cste = value_at_origin / sus(arg_multiplicator)
    return cste * sus(arg_multiplicator * (1.0 - x / x_max))

def soft_one_hot_linspace(
    input: jnp.ndarray,
    *,
    start: float,
    end: float,
    number: int,
    basis: str = None,
    cutoff: bool = None,
    start_zero: bool = None,
    end_zero: bool = None,
):
    r"""Projection on a basis of functions.

    Returns a set of :math:`\{y_i(x)\}_{i=1}^N`,

    .. math::

        y_i(x) = \frac{1}{Z} f_i(x)

    where :math:`x` is the input and :math:`f_i` is the ith basis function.
    :math:`Z` is a constant defined such that,

    .. math::

        \langle \sum_{i=1}^N y_i(x)^2 \rangle_x \approx 1

    See the last plot below.

    Args:
        input (jnp.ndarray): input of shape ``[...]``
        start (float): minimum value span by the basis
        end (float): maximum value span by the basis
        number (int): number of basis functions :math:`N`
        basis (str): type of basis functions, one of ``gaussian``, ``cosine``, ``smooth_finite``, ``fourier``
        cutoff (bool): if ``True``, the basis functions are cutoff at the start and end of the interval
        start_zero (bool): if ``True``, the first basis function is zero at the start of the interval
        end_zero (bool): if ``True``, the last basis function is zero at the end of the interval

    Returns:
        jnp.ndarray: basis functions of shape ``[..., number]``

    Examples:

        .. jupyter-execute::
            :hide-code:

            import jax.numpy as jnp
            import numpy as np
            import e3nn_jax as e3nn
            import matplotlib.pyplot as plt

        .. jupyter-execute::

            bases = ["gaussian", "cosine", "smooth_finite", "fourier"]
            x = np.linspace(-1.0, 2.0, 200)

        .. jupyter-execute::

            fig, axss = plt.subplots(len(bases), 2, figsize=(9, 6), sharex=True, sharey=True)

            for axs, b in zip(axss, bases):
                for ax, c in zip(axs, [True, False]):
                    y = e3nn.soft_one_hot_linspace(x, start=-0.5, end=1.5, number=4, basis=b, cutoff=c)

                    plt.sca(ax)
                    plt.plot(x, y)
                    plt.plot([-0.5]*2, [-2, 2], "k-.")
                    plt.plot([1.5]*2, [-2, 2], "k-.")
                    plt.title(f"{b}" + (" with cutoff" if c else ""))

            plt.ylim(-1, 1.5)
            plt.tight_layout()

        .. jupyter-execute::

            fig, axss = plt.subplots(len(bases), 2, figsize=(9, 6), sharex=True, sharey=True)

            for axs, b in zip(axss, bases):
                for ax, c in zip(axs, [True, False]):
                    y = e3nn.soft_one_hot_linspace(x, start=-0.5, end=1.5, number=4, basis=b, cutoff=c)

                    plt.sca(ax)
                    plt.plot(x, np.sum(y**2, axis=-1))
                    plt.plot([-0.5]*2, [-2, 2], "k-.")
                    plt.plot([1.5]*2, [-2, 2], "k-.")
                    plt.title(f"{b}" + (" with cutoff" if c else ""))

            plt.ylim(0, 2)
            plt.tight_layout()
    """
    if cutoff is not None:
        assert start_zero is None
        assert end_zero is None
        start_zero = cutoff
        end_zero = cutoff

    del cutoff

    if start_zero not in [True, False]:
        raise ValueError("start_zero must be specified")

    if end_zero not in [True, False]:
        raise ValueError("end_zero must be specified")

    if start_zero and end_zero:
        values = jnp.linspace(start, end, number + 2)
        step = values[1] - values[0]
        values = values[1:-1]

    if start_zero and not end_zero:
        values = jnp.linspace(start, end, number + 1)
        step = values[1] - values[0]
        values = values[1:]

    if not start_zero and end_zero:
        values = jnp.linspace(start, end, number + 1)
        step = values[1] - values[0]
        values = values[:-1]

    if not start_zero and not end_zero:
        values = jnp.linspace(start, end, number)
        step = values[1] - values[0]

    diff = (input[..., None] - values) / step

    if basis == "gaussian":
        return jnp.exp(-(diff**2)) / 1.12

    if basis == "cosine":
        return jnp.where((-1.0 < diff) & (diff < 1.0), jnp.cos(jnp.pi / 2 * diff), 0.0)

    if basis == "smooth_finite":
        return 1.14136 * jnp.exp(2.0) * sus(diff + 1.0) * sus(1.0 - diff)

    if basis == "fourier":
        x = (input[..., None] - start) / (end - start)
        if start_zero and end_zero:
            i = jnp.arange(1, number + 1)
            return jnp.where((0.0 < x) & (x < 1.0), jnp.sin(jnp.pi * i * x) / jnp.sqrt(0.25 + number / 2), 0.0)
        elif not start_zero and not end_zero:
            i = jnp.arange(0, number)
            return jnp.cos(jnp.pi * i * x) / jnp.sqrt(0.25 + number / 2)
        else:
            raise ValueError("when using fourier basis, start_zero and end_zero must be the same")

    raise ValueError(f'basis="{basis}" is not a valid entry')

def bessel(x: jnp.ndarray, n: int, x_max: float = 1.0) -> jnp.ndarray:
    r"""Bessel basis functions.

    They obey the following normalization:

    .. math::

        \int_0^c r^2 B_n(r, c) B_m(r, c) dr = \delta_{nm}

    Args:
        x (jnp.ndarray): input of shape ``[...]``
        n (int): number of basis functions
        x_max (float): maximum value of the input

    Returns:
        jnp.ndarray: basis functions of shape ``[..., n]``

    Klicpera, J.; Groß, J.; Günnemann, S. Directional Message Passing for Molecular Graphs; ICLR 2020.
    Equation (7)
    """
    x = x[..., None]
    n = jnp.arange(1, n + 1)
    x_nonzero = jnp.where(x == 0.0, 1.0, x)
    return jnp.sqrt(2.0 / x_max) * jnp.where(
        x == 0,
        n * jnp.pi / x_max,
        jnp.sin(n * jnp.pi / x_max * x_nonzero) / x_nonzero,
    )

def u(p: int, x: jnp.ndarray) -> jnp.ndarray:
    r"""Equivalent to :func:`poly_envelope` with ``n0 = p-1`` and ``n1 = 2``."""
    return 1 - (p + 1) * (p + 2) / 2 * x**p + p * (p + 2) * x ** (p + 1) - p * (p + 1) / 2 * x ** (p + 2)

def _constraint(x: float, derivative: int, degree: int):
    return [0 if derivative > N else factorial(N) // factorial(N - derivative) * x ** (N - derivative) for N in range(degree)]

@lru_cache(maxsize=None)
def solve_polynomial(constraints) -> jnp.ndarray:
    with jax.ensure_compile_time_eval():
        jax_enable_x64 = jax.config.read("jax_enable_x64")
        jax.config.update("jax_enable_x64", True)

        degree = len(constraints)
        A = jnp.array(
            [_constraint(x, derivative, degree) for x, derivative, _ in sorted(constraints)],
            dtype=jnp.float64,
        )
        B = jnp.array([y for _, _, y in sorted(constraints)], dtype=jnp.float64)
        c = jnp.linalg.solve(A, B)

        jax.config.update("jax_enable_x64", jax_enable_x64)
    return jax.jit(lambda x: jnp.polyval(c[::-1], x))

def poly_envelope(n0: int, n1: int, x_max: float = 1.0) -> Callable[[float], float]:
    r"""Polynomial envelope function with ``n0`` and ``n1`` derivatives euqal to 0 at ``x=0`` and ``x=1`` respectively.

    Small documentation available at ``https://mariogeiger.ch/polynomial_envelope_for_gnn.pdf``.
    This is a generalization of :math:`u_p(x)`, it is equivalent to :math:`u_p(x)` when ``n0 = p-1`` and ``n1 = 2``.

    .. jupyter-execute::
        :hide-code:

        import jax.numpy as jnp
        import e3nn_jax as e3nn
        import matplotlib.pyplot as plt

    .. jupyter-execute::

        x = jnp.linspace(0.0, 1.0, 100)
        plt.plot(x, e3nn.poly_envelope(10, 5)(x), label="10, 5")
        plt.plot(x, e3nn.poly_envelope(4, 4)(x), label="4, 4")
        plt.plot(x, e3nn.poly_envelope(1, 2)(x), label="1, 2")
        plt.legend()

    Args:
        n0 (int): number of derivatives equal to 0 at ``x=0``
        n1 (int): number of derivatives equal to 0 at ``x=1``
        x_max (float): maximum value of the input, instead of 1

    Returns:
        callable: polynomial envelope function
    """
    poly = solve_polynomial(
        frozenset(
            {(-0.5, 0, 1.0), (0.5, 0, 0.0)}
            | {(-0.5, derivative, 0.0) for derivative in range(1, n0 + 1)}
            | {(0.5, derivative, 0.0) for derivative in range(1, n1 + 1)}
        )
    )

    def f(x):
        x_small = jnp.where(x < x_max, x, x_max)
        return jnp.where(
            x < x_max,
            poly(x_small / x_max - 0.5),
            0.0,
        )

    return f
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Extract writer application file.

There are multiple functions here write out a CSV to a required location,
based on the data required to be written. The path is variable and the
filename part is expected to be descriptive of the method of fetching
from the API.

Rows are appended so that the data can be continously added to an existing
file. This for when stopping and resuming the extract process, switching
between various search queries, or when periodically writing out data every 15
minutes (to reduce the number of tweets held in memory at a time which would
slow performance and be a risk of losing data on the extract script aborting).

To flatten pages of tweets into a flat list of tweets, see how to make a flat
list out of a list of lists:
    https://stackoverflow.com/questions/952914/making-a-flat-list-out-of-list-of-lists-in-python

Handling universal newlines in Python 3:
    https://softwareengineering.stackexchange.com/questions/298677/why-is-universal-newlines-mode-deprecated-in-python
"""
import csv
import datetime
import logging
import os
import pytz

logger = logging.getLogger("lib.extract.writer")

# Column name groupings for writing out to a CSV. The column names within
# each list have been arranged in the order to be written out.
PROFILE_COLUMNS = ['profileGuid', 'screenName', 'name', 'description',
                  'location', 'imageUrl', 'followersCount', 'statusesCount',
                  'verified']
TWEET_COLUMNS = ['tweetGuid', 'createdAt', 'message', 'favoriteCount',
                 'retweetCount', 'inReplyToTweetGuid', 'inReplyToProfileGuid']
METADATA_COLUMNS = ['campaignName', 'modified']

def convertToOutrow(campaignName, modified, fetchedProfile=None,
                    fetchedTweet=None):
    """Convert fetched Twitter data and metadata as a row for a CSV writer.

    Supply one or both of the `fetchTweet` and `fetchProfile` parameters to
    include the input data in the returned dict.

    @param campaignName: Name of the campaign associated with tweet. If this
        is given as `None`, then it is written as an empty string.
    @param modified: Fetch time of the data, as a datetime.datetime object.
    @param fetchProfile: Optional tweepy tweet author object as fetched
        from the Twitter API. If provided, add to response dict.
    @param fetchedTweet: Optional tweepy tweet object as fetched from the
        Twitter API. If provided, add to response dict.

    @return: dict object of fields around a profile and/or profile object.
        All unicode values are converted to str, since the csv.DictWriter
        object only handles str types.
    """
    outData = {}

    if fetchedProfile:
        profileData = {
            'profileGuid':          fetchedProfile.id,
            'screenName':           fetchedProfile.screen_name,
            'name':                 fetchedProfile.name,
            'description':          fetchedProfile.description,
            'location':             fetchedProfile.location,
            'imageUrl':             fetchedProfile.profile_image_url_https,
            'followersCount':       fetchedProfile.followers_count,
            'statusesCount':        fetchedProfile.statuses_count
        }
        outData.update(profileData)

    if fetchedTweet:
        # Tweepy has already created a datetime string into a datetime object
        # for us, but it is unaware of the timezone. We know that the timezone
        # is always given as UTC+0000 regardless of where the tweet was made,
        # so we can set the tzinfo safely.
        awareTime = fetchedTweet.created_at.replace(tzinfo=pytz.UTC)

        # Assume extended mode was requested from the API and the full_text
        # field will be returned, but fall back to standard mode.
        try:
            text = fetchedTweet.full_text
        except AttributeError:
            text = fetchedTweet.text

        tweetData = {
            'tweetGuid':            fetchedTweet.id,
            'createdAt':            str(awareTime),
            'message':              text,
            'favoriteCount':        fetchedTweet.favorite_count,
            'retweetCount':         fetchedTweet.retweet_count,
            'inReplyToTweetGuid':   fetchedTweet.in_reply_to_status_id,
            'inReplyToProfileGuid': fetchedTweet.in_reply_to_user_id
        }
        outData.update(tweetData)

    metaData = {
        'campaignName':         campaignName,
        'modified':             str(modified)
    }
    outData.update(metaData)

    return {k: (v.encode('utf-8') if type(v) is unicode else v)
            for k, v in outData.iteritems()}

def writeProfilesAndTweets(outPath, outPages, campaignName=None,
                           modified=None):
    """Format received pages of Twitter data and append rows of data to a CSV.

    @param outPath: Path of CSV file to write to.
    @param outPages: Pages of tweepy tweet objects to be written out.
    @param campaignName: Optional name of campaign to associate tweet record
        with. By default, this will be written as an empty string value in the
        output CSV.
    @param modified: Optional fetch time of the data, as a datetime.datetime
        object. If not specified, the time at the start of this function
        will be used in all rows.

    @return count: Count of rows written out.
    """
    startTime = datetime.datetime.now()

    if modified is None:
        modified = datetime.datetime.now()

    # TODO: Write in logic here to use different iteration and arguments
    # depending on the requirements.
    outRows = [
        convertToOutrow(
            campaignName,
            modified,
            fetchedTweet=tweet,
            fetchedProfile=tweet.author
        )
        for page in outPages for tweet in page
    ]

    fileExists = os.path.exists(outPath)

    with open(outPath, "a") as fOut:
        # TODO: Write in logic to determine this based on arguments.
        fieldNames = PROFILE_COLUMNS + TWEET_COLUMNS + METADATA_COLUMNS
        csvWriter = csv.DictWriter(fOut, fieldNames, lineterminator="\n")
        if not fileExists:
            # Only write the header if the file did not exist before.
            csvWriter.writeheader()
        csvWriter.writerows(outRows)

    count = len(outRows)
    filename = os.path.basename(outPath)
    duration = datetime.datetime.now() - startTime

    logger.info(
        "Wrote {count:,d} rows to: {filename} in {duration:3,.2f}s"
                .format(
                count=count,
                filename=filename,
                duration=duration.total_seconds()
            )
    )

    return count
<commit_msg>style: Update alignment in writer.py docs: Update comment in writer.py<commit_after># -*- coding: utf-8 -*-
"""
Extract writer application file.

There are multiple functions here write out a CSV to a required location,
based on the data required to be written. The path is variable and the
filename part is expected to be descriptive of the method of fetching
from the API.

Rows are appended so that the data can be continously added to an existing
file. This for when stopping and resuming the extract process, switching
between various search queries, or when periodically writing out data every 15
minutes (to reduce the number of tweets held in memory at a time which would
slow performance and be a risk of losing data on the extract script aborting).

To flatten pages of tweets into a flat list of tweets, see how to make a flat
list out of a list of lists:
    https://stackoverflow.com/questions/952914/making-a-flat-list-out-of-list-of-lists-in-python

Handling universal newlines in Python 3:
    https://softwareengineering.stackexchange.com/questions/298677/why-is-universal-newlines-mode-deprecated-in-python
"""
import csv
import datetime
import logging
import os
import pytz

logger = logging.getLogger("lib.extract.writer")

# Column name groupings for writing out to a CSV. The column names within
# each list have been arranged in the order to be written out.
PROFILE_COLUMNS = ['profileGuid', 'screenName', 'name', 'description',
                  'location', 'imageUrl', 'followersCount', 'statusesCount',
                  'verified']
TWEET_COLUMNS = ['tweetGuid', 'createdAt', 'message', 'favoriteCount',
                 'retweetCount', 'inReplyToTweetGuid', 'inReplyToProfileGuid']
METADATA_COLUMNS = ['campaignName', 'modified']

def convertToOutrow(campaignName, modified, fetchedProfile=None,
                    fetchedTweet=None):
    """Convert fetched Twitter data and metadata as a row for a CSV writer.

    Supply one or both of the `fetchTweet` and `fetchProfile` parameters to
    include the input data in the returned dict.

    @param campaignName: Name of the campaign associated with tweet. If this
        is given as `None`, then it is written as an empty string.
    @param modified: Fetch time of the data, as a datetime.datetime object.
    @param fetchProfile: Optional tweepy tweet author object as fetched
        from the Twitter API. If provided, add to response dict.
    @param fetchedTweet: Optional tweepy tweet object as fetched from the
        Twitter API. If provided, add to response dict.

    @return: dict object of fields around a profile and/or profile object.
        All unicode values are converted to str, since the csv.DictWriter
        object only handles str types.
    """
    outData = {}

    if fetchedProfile:
        profileData = {
            'profileGuid':          fetchedProfile.id,
            'screenName':           fetchedProfile.screen_name,
            'name':                 fetchedProfile.name,
            'description':          fetchedProfile.description,
            'location':             fetchedProfile.location,
            'imageUrl':             fetchedProfile.profile_image_url_https,
            'followersCount':       fetchedProfile.followers_count,
            'statusesCount':        fetchedProfile.statuses_count
        }
        outData.update(profileData)

    if fetchedTweet:
        # Tweepy has already created a datetime string into a datetime object
        # for us, but it is unaware of the timezone. We know that the timezone
        # is always given as UTC+0000 regardless of where the tweet was made,
        # so we can set the tzinfo safely.
        awareTime = fetchedTweet.created_at.replace(tzinfo=pytz.UTC)

        # Assume extended mode was requested from the API and the full_text
        # field will be returned, but fall back to standard mode.
        try:
            text = fetchedTweet.full_text
        except AttributeError:
            text = fetchedTweet.text

        tweetData = {
            'tweetGuid':            fetchedTweet.id,
            'createdAt':            str(awareTime),
            'message':              text,
            'favoriteCount':        fetchedTweet.favorite_count,
            'retweetCount':         fetchedTweet.retweet_count,
            'inReplyToTweetGuid':   fetchedTweet.in_reply_to_status_id,
            'inReplyToProfileGuid': fetchedTweet.in_reply_to_user_id
        }
        outData.update(tweetData)

    metaData = {
        'campaignName':         campaignName,
        'modified':             str(modified)
    }
    outData.update(metaData)

    return {k: (v.encode('utf-8') if type(v) is unicode else v)
            for k, v in outData.iteritems()}

def writeProfilesAndTweets(outPath, outPages, campaignName=None,
                           modified=None):
    """Format received pages of Twitter data and append rows of data to a CSV.

    @param outPath: Path of CSV file to write to.
    @param outPages: Pages of tweepy tweet objects to be written out.
    @param campaignName: Optional name of campaign to associate tweet record
        with. By default, this will be written as an empty string value in the
        output CSV.
    @param modified: Optional fetch time of the data, as a datetime.datetime
        object. If not specified, the time at the start of this function
        will be used in all rows.

    @return count: Count of rows written out.
    """
    startTime = datetime.datetime.now()

    if modified is None:
        modified = datetime.datetime.now()

    # TODO: Write in logic here to use different iteration and arguments
    # depending on the requirements, for when it is not tweets and authors.
    outRows = [
        convertToOutrow(
            campaignName,
            modified,
            fetchedTweet=tweet,
            fetchedProfile=tweet.author
        )
        for page in outPages for tweet in page
    ]

    fileExists = os.path.exists(outPath)

    with open(outPath, "a") as fOut:
        # TODO: Write in logic to determine this based on arguments.
        fieldNames = PROFILE_COLUMNS + TWEET_COLUMNS + METADATA_COLUMNS
        csvWriter = csv.DictWriter(
            fOut,
            fieldNames,
            lineterminator="\n"
        )
        if not fileExists:
            # Only write the header if the file did not exist before.
            csvWriter.writeheader()
        csvWriter.writerows(outRows)

    count = len(outRows)
    filename = os.path.basename(outPath)
    duration = datetime.datetime.now() - startTime

    logger.info(
        "Wrote {count:,d} rows to: {filename} in {duration:3,.2f}s"
        .format(
            count=count,
            filename=filename,
            duration=duration.total_seconds()
        )
    )

    return count
<|endoftext|>


<commit_before># coding=utf-8
# Licensed Materials - Property of IBM
# Copyright IBM Corp. 2017
"""
Overview
--------
Access to the IBM Streams execution context.

A Streams application runs distributed or standalone.

Distributed
-----------
Distributed is used when an application is submitted
to the Streaming Analytics service on IBM Bluemix
or a IBM Streams distributed instance.

With distributed a running application is a `job` that
contains one or more processing elements (PEs). A PE
corresponds to a Linux operating system process.
The PEs in a job may be distributed across the
hosts in the Streams instance.

Standalone
----------
Standalone is a mode where the complete application is run
as a single PE (process) outside of a Streams instance.

Standalone is typically used for ad-hoc testing of an
application.

Execution Context
-----------------
This module (`streamsx.exec`) provides access to the execution
context when Python code is running in a Streams application.

Access is only supported when running:
 * Python 3.5
 * Streams 4.2 or later

This module may be used by Python functions or classes used
in a `Topology` or decorated SPL operators.

Certain functionality is only available when a Python class is
being invoked in a Streams application.

"""

import enum
import threading

try:
    import _streamsx_ec as _ec
    _supported = True
except ImportError:
    _supported = False

def _check():
    if not _supported:
        raise NotImplementedError("Access to the execution context requires Python 3.5 and Streams 4.2 or later")

def job_id():
    """
    Return the job identifier.
    """
    _check()
    return _ec.job_id()

def pe_id():
    """
    Return the PE identifier.
    """
    _check()
    return _ec.pe_id()

def channel(obj):
    """
    Return the parallel region global channel number `obj` is executing in.

    The channel number is in the range of 0 to ``max_channel(obj)``.

    When the parallel region is not nested this is the same value
    as ``local_channel(obj)``.

    If the parallel region is nested the value will be between
    zero and ``(width*N - 1)`` where N is the number of times the
    parallel region has been replicated due to nesting.

    Args:
        obj: Instance of a class executing as an SPL Python operator.

    Returns:
        int: Parallel region global channel number or -1 if not located in a parallel region.
    """
    return _ec.channel(_get_opc(obj))

def local_channel(obj):
    """
    Return the parallel region local channel number `obj` is executing in.

    The channel number is in the range of zero to ``local_max_channel(obj)``.

    Args:
        obj: Instance of a class executing as an SPL Python operator.

    Returns:
        int: Parallel region local channel number or -1 if not located in a parallel region.
    """
    return _ec.local_channel(_get_opc(obj))

def max_channels(obj):
    """
    Return the global maximum number of channels for the parallel
    region `obj` is executing in.

    When the parallel region is not nested this is the same value
    as ``local_max_channels(obj)``.

    If the parallel region is nested the value will be
    ``(width*N)`` where N is the number of times the
    parallel region has been replicated due to nesting.

    Args:
        obj: Instance of a class executing as an SPL Python operator.
    Returns:
        int: Parallel region global maximum number of channels or 0 if not located in a parallel region.
    """
    return _ec.max_channels(_get_opc(obj))

def local_max_channels(obj):
    """
    Return the local maximum number of channels for the parallel
    region `obj` is executing in.

    The maximum number of channels corresponds to the width of the region.

    Args:
        obj: Instance of a class executing as an SPL Python operator.
    Returns:
        int: Parallel region local maximum number of channels or 0 if not located in a parallel region.
    """
    return _ec.local_max_channels(_get_opc(obj))

@enum.unique
class MetricKind(enum.Enum):
    """
    Enumeration for the kind of a metric.

    The kind of the metric only indicates the behavior of value,
    it does not impose any semantics on the value.
    The kind is typically used by tooling applications.

    """
    Gauge = 0
    """
    A gauge metric observes a value that is continuously variable with time.
    """
    Counter = 1
    """
    A counter metric observes a value that represents a count of an occurrence.
    """
    Time = 2
    """
    A time metric represents a point in time or duration.
    The recommended unit of time is milliseconds, using the standard
    epoch of 00:00:00 Coordinated Universal Time (UTC),
    Thursday, 1 January 1970 to represent a point in time.
    """

class CustomMetric(object):
    """
    Create a custom metric.

    A custom metric holds a 64 bit signed integer value that represents
    a `Counter`, `Gauge` or `Time` metric.
    Custom metrics are exposed through the IBM Streams monitoring APIs.

    Args:
        obj: Instance of a class executing as an SPL Python operator.
        name(str): Name of the custom metric.
        kind(MetricKind): Kind of the metric.
        description(str): Description of the metric.
        initialValue: Initial value of the metric.
    """
    def __init__(self, obj, name, description=None, kind=MetricKind.Counter, initialValue=0):
        if kind in MetricKind.__members__:
            kind = MetricKind.__members__[kind]
        elif kind not in MetricKind.__members__.values():
            raise TypeError("kind is required to be MetricKind:" + kind)
        if description is None:
            description=name + ":" + kind.name
        self.name = str(name)
        self.kind = kind
        self.description = str(description)
        args = (_get_opc(obj), self.name, self.description, self.kind.value, int(initialValue))
        self.__ptr = _ec.create_custom_metric(args)

    @property
    def value(self):
        """
        Current value of the metric.
        """
        return _ec.metric_get(self.__ptr)

    @value.setter
    def value(self, value):
        """
        Set the current value of the metric.
        """
        args = (self.__ptr, int(value))
        _ec.metric_set(args)

    def __str__(self):
        return "{0}({1}):{2}".format(self.name, self.kind.name,self.value)

    def __iadd__(self, other):
        """
        Increment the current value of the metric.
        """
        args = (self.__ptr, int(other))
        _ec.metric_inc(args)
        return self

    def __int__(self):
        return self.value

####################
# internal functions
####################

# Thread local of operator pointers during
# operator class initialization
_opptrs = threading.local()

# Sets the operator pointer as a thread
# local to allow access from an operator's
# class __init__ method.
def _set_opc(opc):
    _opptrs._opc = opc

# Clear the operator pointer from the
# thread local
def _clear_opc():
    _opptrs._opc = None

# Save the opc in the operator class
# (getting it from the thread local)
def _save_opc(obj):
    _opptrs.obj = obj
    if hasattr(_opptrs, '_opc'):
       opc = _opptrs._opc
       if opc is not None:
           obj._streamsx_ec_op = opc

def _get_opc(obj):
    _check()
    try:
        return obj._streamsx_ec_op
    except AttributeError:
        try:
            opc = _opptrs._opc
            if opc is not None:
                return opc
        except AttributeError:
             pass
        raise AssertionError("InternalError")
<commit_msg>Add example<commit_after># coding=utf-8
# Licensed Materials - Property of IBM
# Copyright IBM Corp. 2017
"""
Overview
--------
Access to the IBM Streams execution context.

A Streams application runs distributed or standalone.

Distributed
-----------
Distributed is used when an application is submitted
to the Streaming Analytics service on IBM Bluemix
or a IBM Streams distributed instance.

With distributed a running application is a `job` that
contains one or more processing elements (PEs). A PE
corresponds to a Linux operating system process.
The PEs in a job may be distributed across the
hosts in the Streams instance.

Standalone
----------
Standalone is a mode where the complete application is run
as a single PE (process) outside of a Streams instance.

Standalone is typically used for ad-hoc testing of an
application.

Execution Context
-----------------
This module (`streamsx.exec`) provides access to the execution
context when Python code is running in a Streams application.

Access is only supported when running:
 * Python 3.5
 * Streams 4.2 or later

This module may be used by Python functions or classes used
in a `Topology` or decorated SPL operators.

Most functionality is only available when a Python class is
being invoked in a Streams application.

"""

import enum
import threading

try:
    import _streamsx_ec as _ec
    _supported = True
except ImportError:
    _supported = False

def _check():
    if not _supported:
        raise NotImplementedError("Access to the execution context requires Python 3.5 and Streams 4.2 or later")

def job_id():
    """
    Return the job identifier.
    """
    _check()
    return _ec.job_id()

def pe_id():
    """
    Return the PE identifier.
    """
    _check()
    return _ec.pe_id()

def channel(obj):
    """
    Return the parallel region global channel number `obj` is executing in.

    The channel number is in the range of 0 to ``max_channel(obj)``.

    When the parallel region is not nested this is the same value
    as ``local_channel(obj)``.

    If the parallel region is nested the value will be between
    zero and ``(width*N - 1)`` where N is the number of times the
    parallel region has been replicated due to nesting.

    Args:
        obj: Instance of a class executing within Streams.

    Returns:
        int: Parallel region global channel number or -1 if not located in a parallel region.
    """
    return _ec.channel(_get_opc(obj))

def local_channel(obj):
    """
    Return the parallel region local channel number `obj` is executing in.

    The channel number is in the range of zero to ``local_max_channel(obj)``.

    Args:
        obj: Instance of a class executing within Streams.

    Returns:
        int: Parallel region local channel number or -1 if not located in a parallel region.
    """
    return _ec.local_channel(_get_opc(obj))

def max_channels(obj):
    """
    Return the global maximum number of channels for the parallel
    region `obj` is executing in.

    When the parallel region is not nested this is the same value
    as ``local_max_channels(obj)``.

    If the parallel region is nested the value will be
    ``(width*N)`` where N is the number of times the
    parallel region has been replicated due to nesting.

    Args:
        obj: Instance of a class executing within Streams.
    Returns:
        int: Parallel region global maximum number of channels or 0 if not located in a parallel region.
    """
    return _ec.max_channels(_get_opc(obj))

def local_max_channels(obj):
    """
    Return the local maximum number of channels for the parallel
    region `obj` is executing in.

    The maximum number of channels corresponds to the width of the region.

    Args:
        obj: Instance of a class executing within Streams.
    Returns:
        int: Parallel region local maximum number of channels or 0 if not located in a parallel region.
    """
    return _ec.local_max_channels(_get_opc(obj))

@enum.unique
class MetricKind(enum.Enum):
    """
    Enumeration for the kind of a metric.

    The kind of the metric only indicates the behavior of value,
    it does not impose any semantics on the value.
    The kind is typically used by tooling applications.

    """
    Gauge = 0
    """
    A gauge metric observes a value that is continuously variable with time.
    """
    Counter = 1
    """
    A counter metric observes a value that represents a count of an occurrence.
    """
    Time = 2
    """
    A time metric represents a point in time or duration.
    The recommended unit of time is milliseconds, using the standard
    epoch of 00:00:00 Coordinated Universal Time (UTC),
    Thursday, 1 January 1970 to represent a point in time.
    """

class CustomMetric(object):
    """
    Create a custom metric.

    A custom metric holds a 64 bit signed integer value that represents
    a `Counter`, `Gauge` or `Time` metric.

    Custom metrics are exposed through the IBM Streams monitoring APIs.

    Args:
        obj: Instance of a class executing within Streams.
        name(str): Name of the custom metric.
        kind(MetricKind): Kind of the metric.
        description(str): Description of the metric.
        initialValue: Initial value of the metric.

    Examples:

    Simple example used as an instance to ``Stream.filter``::

	class MyF:
	    def __init__(self, substring):
                self.substring = substring
                pass

            def __call__(self, tuple):
                if self.substring in str(tuple)
                    self.my_metric += 1
                return True

            def __getstate__(self):
                # Remove metric from saved state.
                state = self.__dict__.copy()
                if 'my_metric' in state:
                    del state['my_metric']
                return state

            def __setstate__(self, state):
                # Create the metric after the instance is depickled.
                self.__dict__.update(state)
                self.my_metric = ec.CustomMetric(self, "count_" + self.substring)

    """
    def __init__(self, obj, name, description=None, kind=MetricKind.Counter, initialValue=0):
        if kind in MetricKind.__members__:
            kind = MetricKind.__members__[kind]
        elif kind not in MetricKind.__members__.values():
            raise TypeError("kind is required to be MetricKind:" + kind)
        if description is None:
            description=name + ":" + kind.name
        self.name = str(name)
        self.kind = kind
        self.description = str(description)
        args = (_get_opc(obj), self.name, self.description, self.kind.value, int(initialValue))
        self.__ptr = _ec.create_custom_metric(args)

    @property
    def value(self):
        """
        Current value of the metric.
        """
        return _ec.metric_get(self.__ptr)

    @value.setter
    def value(self, value):
        """
        Set the current value of the metric.
        """
        args = (self.__ptr, int(value))
        _ec.metric_set(args)

    def __str__(self):
        return "{0}({1}):{2}".format(self.name, self.kind.name,self.value)

    def __iadd__(self, other):
        """
        Increment the current value of the metric.
        """
        args = (self.__ptr, int(other))
        _ec.metric_inc(args)
        return self

    def __int__(self):
        return self.value

####################
# internal functions
####################

# Thread local of operator pointers during
# operator class initialization
_opptrs = threading.local()

# Sets the operator pointer as a thread
# local to allow access from an operator's
# class __init__ method.
def _set_opc(opc):
    _opptrs._opc = opc

# Clear the operator pointer from the
# thread local
def _clear_opc():
    _opptrs._opc = None

# Save the opc in the operator class
# (getting it from the thread local)
def _save_opc(obj):
    _opptrs.obj = obj
    if hasattr(_opptrs, '_opc'):
       opc = _opptrs._opc
       if opc is not None:
           obj._streamsx_ec_op = opc

def _get_opc(obj):
    _check()
    try:
        return obj._streamsx_ec_op
    except AttributeError:
        try:
            opc = _opptrs._opc
            if opc is not None:
                return opc
        except AttributeError:
             pass
        raise AssertionError("InternalError")
<|endoftext|>


<commit_before>from siuba.siu import Symbolic, strip_symbolic
from siuba.spec.series import spec
from .helpers import data_frame, assert_equal_query, backend_pandas, SqlBackend, PandasBackend
import pytest
# TODO: dot, corr, cov

from siuba import filter, mutate, summarize, group_by
from pandas.testing import assert_frame_equal, assert_series_equal
import numpy as np
import pandas as pd
import pkg_resources

def get_action_kind(spec_entry):
    action = spec_entry['action']
    return action.get('kind', action.get('status')).title()

def filter_on_result(spec, types):
    return [k for k,v in spec.items() if v['action'].get('kind', v['action'].get('status')).title() in types]

SPEC_IMPLEMENTED = filter_on_result(spec, {"Agg", "Elwise", "Window"})
SPEC_NOTIMPLEMENTED = filter_on_result(spec, {"Singleton", "Wontdo", "Todo", "Maydo"})
SPEC_AGG = filter_on_result(spec, {"Agg"})

_ = Symbolic()

@pytest.fixture(params = tuple(SPEC_IMPLEMENTED))
def entry(request):
    # NOTE: the sole purpose of putting in a fixture is so pytest line output
    #       is very easy to read. (e.g. pytest -v --tb=line)
    key = request.param
    yield spec[key]

@pytest.fixture(params = tuple(SPEC_AGG))
def agg_entry(request):
    key = request.param
    yield spec[key]

@pytest.fixture(params = tuple(SPEC_NOTIMPLEMENTED))
def notimpl_entry(request):
    key  = request.param
    yield spec[key]

def assert_src_array_equal(src, dst):
    if isinstance(src, np.ndarray):
        assert np.array_equal(src, dst)
    elif isinstance(src, pd.DataFrame):
        assert_frame_equal(src, dst)
    elif isinstance(src, pd.Series):
        assert_series_equal(src, dst, check_names = False)
    else:
        assert src == dst

# Data ========================================================================
data_dt = data_frame(
    g = ['a', 'a', 'b', 'b'],
    x = pd.to_datetime(["2019-01-01 01:01:01", "2020-04-08 02:02:02", "2021-07-15 03:03:03", "2022-10-22 04:04:04"])
    )

data_str = data_frame(
    g = ['a', 'a', 'b', 'b'],
    x = ['abc', 'cde', 'fg', 'h']
    )

data_bool = data_frame(
    g = ['a', 'a', 'b', 'b'],
    x = [True, False, True, False],
    y = [True, True, False, False]
        )

data_cat = data_frame(
    g = ['a', 'a', 'b', 'b'],
    x = pd.Categorical(['abc', 'cde', 'fg', 'h'])
    )

data_sparse = data_frame(
    g = ['a', 'a', 'b', 'b'],
    x = pd.Series([1, 2, 3, 4], dtype = "Sparse")
    )

data_default = data_frame(
    g = ['a', 'a', 'a', 'b', 'b', 'b'],
    x = [10, 11, 11, 13, 13, 13],
    y = [1,2,3,4,5,6]
    )

DATA = data = {
    'dt': data_dt,
    'str': data_str,
    None: data_default,
    'bool': data_bool,
    'cat': data_cat,
    'sparse': data_sparse

}

def get_spec_no_mutate(entry, backend):
    return "no_mutate" in entry['backends'].get(backend, {}).get('flags', [])

def get_spec_backend_status(entry, backend):
    return entry['backends'].get(backend, {}).get('status')

def get_spec_sql_type(entry, backend):
    return entry['backends'].get(backend, {}).get('result_type')

def get_data(entry, data, backend = None):

    req_bool = entry['action'].get('input_type') == 'bool'

    # pandas is forgiving to bool inputs
    if isinstance(backend, PandasBackend):
        req_bool = False

    return data['bool'] if req_bool else data[entry['accessor']]

def test_missing_implementation(entry, backend):
    # Check whether test should xfail, skip, or -------------------------------
    backend_status = get_spec_backend_status(entry, backend.name)

    # case: Needs to be implmented
    # TODO(table): uses xfail
    if backend_status == "todo":
        pytest.xfail("TODO: impelement this translation")

    # case: Can't be used in a mutate (e.g. a SQL ordered set aggregate function)
    # TODO(table): no_mutate
    if get_spec_no_mutate(entry, backend.name):
        pytest.skip("Spec'd failure")

    # case: won't be implemented
    if get_spec_backend_status(entry, backend.name) == "wontdo":
        pytest.skip()

def get_df_expr(entry):
    str_expr = str(entry['expr_frame'])
    call_expr = strip_symbolic(eval(str_expr, {'_': _}))

    return str_expr, call_expr

def cast_result_type(entry, backend, ser):
    sql_type = get_spec_sql_type(entry, backend.name)
    if isinstance(backend, SqlBackend) and sql_type == 'float':
        return ser.astype('float')

    return ser

# Tests =======================================================================

def test_series_against_call(entry):
    if entry['action']['kind'] == "window":
        pytest.skip()

    df = data[entry['accessor']]
    # TODO: once reading from yaml, no need to repr
    str_expr = str(entry['expr_series'])

    call_expr = strip_symbolic(eval(str_expr, {'_': _}))
    res = call_expr(df.x)

    dst = eval(str_expr, {'_': df.x})

    assert res.__class__ is dst.__class__
    assert_src_array_equal(res, dst)

def test_frame_expr(entry):
    # TODO: remove this test, not checking anything new
    df = data[entry['accessor']]
    # TODO: once reading from yaml, no need to repr
    str_expr = str(entry['expr_frame'])

    call_expr = strip_symbolic(eval(str_expr, {'_': _}))
    res = call_expr(df)

    dst = eval(str_expr, {'_': df})

    assert res.__class__ is dst.__class__
    assert_src_array_equal(res, dst)

def test_pandas_grouped_frame_fast_not_implemented(notimpl_entry):
    from siuba.experimental.pd_groups.dialect import fast_mutate
    gdf = data[notimpl_entry['accessor']].groupby('g')

    # TODO: once reading from yaml, no need to repr
    str_expr = str(notimpl_entry['expr_frame'])
    call_expr = strip_symbolic(eval(str_expr, {'_': _}))

    if notimpl_entry['action']['status'] in ["todo", "maydo", "wontdo"] and notimpl_entry["is_property"]:
        pytest.xfail()

    with pytest.raises(NotImplementedError):
        res = fast_mutate(gdf, result = call_expr)

#@backend_pandas
@pytest.mark.skip_backend('sqlite')
def test_frame_mutate(skip_backend, backend, entry):
    test_missing_implementation(entry, backend)

    # Prepare input data ------------------------------------------------------
    # case: inputs must be boolean
    crnt_data = get_data(entry, DATA, backend)
    df = backend.load_df(crnt_data)

    # Execute mutate ----------------------------------------------------------
    str_expr, call_expr = get_df_expr(entry)

    # Run test for equality w/ ungrouped pandas ----
    dst = crnt_data.assign(result = call_expr(crnt_data))
    dst['result'] = cast_result_type(entry, backend, dst['result'])

    assert_equal_query(
            df,
            mutate(result = call_expr),
            dst
            )

    # Run test for equality w/ grouped pandas ----
    g_dst = crnt_data.groupby('g').apply(lambda d: d.assign(result = call_expr)).reset_index(drop = True)
    g_dst['result'] = cast_result_type(entry, backend, g_dst['result'])
    assert_equal_query(
            df,
            group_by(_.g) >> mutate(result = call_expr),
            g_dst
            )

def test_pandas_grouped_frame_fast_mutate(entry):
    from siuba.experimental.pd_groups.dialect import fast_mutate, DataFrameGroupBy
    gdf = get_data(entry, DATA).groupby('g')

    # Execute mutate ----------------------------------------------------------
    str_expr, call_expr = get_df_expr(entry)

    res = fast_mutate(gdf, result = call_expr)
    dst = mutate(gdf, result = call_expr)

    # TODO: apply mark to skip failing tests, rather than downcast
    # pandas grouped aggs, when not using cython, _try_cast back to original type
    # but since mutate uses apply, it doesn't :/. Currently only affects median func.
    dst_obj = dst.obj
    if str_expr == '_.x.median()':
        dst_obj['result'] = gdf._try_cast(dst_obj['result'], gdf.x.obj)

    assert isinstance(dst, DataFrameGroupBy)
    assert_frame_equal(res.obj, dst_obj)

@pytest.mark.skip_backend('sqlite')
def test_frame_summarize(skip_backend, backend, agg_entry):
    entry = agg_entry
    test_missing_implementation(entry, backend)

    # Prepare input data ------------------------------------------------------
    # case: inputs must be boolean
    crnt_data = get_data(entry, DATA, backend)
    df = backend.load_df(crnt_data)

    # Execute mutate ----------------------------------------------------------
    str_expr, call_expr = get_df_expr(entry)

    dst = data_frame(result = call_expr(crnt_data))

    # Process output ----------------------------------------------------------
    # case: output is of a different type than w/ pandas
    dst['result'] = cast_result_type(entry, backend, dst['result'])

    # Run test for equality w/ pandas ----
    # otherwise, verify returns same result as mutate
    assert_equal_query(
            df,
            summarize(result = call_expr),
            dst
            )

    dst_g = crnt_data.groupby('g').apply(call_expr).reset_index().rename(columns = {0: 'result'})
    assert_equal_query(
            df,
            group_by(_.g) >> summarize(result = call_expr),
            dst_g
            )

def test_pandas_grouped_frame_fast_summarize(agg_entry):
    from siuba.experimental.pd_groups.dialect import fast_summarize, DataFrameGroupBy
    gdf = get_data(agg_entry, DATA).groupby('g')

    # Execute summarize ----------------------------------------------------------
    str_expr, call_expr = get_df_expr(agg_entry)

    res = fast_summarize(gdf, result = call_expr)
    dst = summarize(gdf, result = call_expr)

    # TODO: apply mark to skip failing tests, rather than downcast
    # pandas grouped aggs, when not using cython, _try_cast back to original type
    # but since summarize uses apply, it doesn't :/. Currently only affects median func.
    if str_expr == '_.x.median()':
        dst['result'] = gdf._try_cast(dst['result'], gdf.x.obj)

    assert_frame_equal(res, dst)

# Edge Cases ==================================================================

def test_frame_set_aggregates_postgresql():
    # TODO: probably shouldn't be creating backend here
    backend = SqlBackend("postgresql")
    dfs = backend.load_df(data[None])

    expr = _.x.quantile(.75)
    assert_equal_query(
            dfs,
            group_by(_.g) >> summarize(result = expr),
            data_frame(g = ['a', 'b'], result = [11., 13.])
            )

<commit_msg>tests: remove use of pandas old method _try_cast<commit_after>from siuba.siu import Symbolic, strip_symbolic
from siuba.spec.series import spec
from .helpers import data_frame, assert_equal_query, backend_pandas, SqlBackend, PandasBackend
import pytest
# TODO: dot, corr, cov

from siuba import filter, mutate, summarize, group_by
from pandas.testing import assert_frame_equal, assert_series_equal
import numpy as np
import pandas as pd
import pkg_resources

def get_action_kind(spec_entry):
    action = spec_entry['action']
    return action.get('kind', action.get('status')).title()

def filter_on_result(spec, types):
    return [k for k,v in spec.items() if v['action'].get('kind', v['action'].get('status')).title() in types]

SPEC_IMPLEMENTED = filter_on_result(spec, {"Agg", "Elwise", "Window"})
SPEC_NOTIMPLEMENTED = filter_on_result(spec, {"Singleton", "Wontdo", "Todo", "Maydo"})
SPEC_AGG = filter_on_result(spec, {"Agg"})

_ = Symbolic()

@pytest.fixture(params = tuple(SPEC_IMPLEMENTED))
def entry(request):
    # NOTE: the sole purpose of putting in a fixture is so pytest line output
    #       is very easy to read. (e.g. pytest -v --tb=line)
    key = request.param
    yield spec[key]

@pytest.fixture(params = tuple(SPEC_AGG))
def agg_entry(request):
    key = request.param
    yield spec[key]

@pytest.fixture(params = tuple(SPEC_NOTIMPLEMENTED))
def notimpl_entry(request):
    key  = request.param
    yield spec[key]

def assert_src_array_equal(src, dst):
    if isinstance(src, np.ndarray):
        assert np.array_equal(src, dst)
    elif isinstance(src, pd.DataFrame):
        assert_frame_equal(src, dst)
    elif isinstance(src, pd.Series):
        assert_series_equal(src, dst, check_names = False)
    else:
        assert src == dst

# Data ========================================================================
data_dt = data_frame(
    g = ['a', 'a', 'b', 'b'],
    x = pd.to_datetime(["2019-01-01 01:01:01", "2020-04-08 02:02:02", "2021-07-15 03:03:03", "2022-10-22 04:04:04"])
    )

data_str = data_frame(
    g = ['a', 'a', 'b', 'b'],
    x = ['abc', 'cde', 'fg', 'h']
    )

data_bool = data_frame(
    g = ['a', 'a', 'b', 'b'],
    x = [True, False, True, False],
    y = [True, True, False, False]
        )

data_cat = data_frame(
    g = ['a', 'a', 'b', 'b'],
    x = pd.Categorical(['abc', 'cde', 'fg', 'h'])
    )

data_sparse = data_frame(
    g = ['a', 'a', 'b', 'b'],
    x = pd.Series([1, 2, 3, 4], dtype = "Sparse")
    )

data_default = data_frame(
    g = ['a', 'a', 'a', 'b', 'b', 'b'],
    x = [10, 11, 11, 13, 13, 13],
    y = [1,2,3,4,5,6]
    )

DATA = data = {
    'dt': data_dt,
    'str': data_str,
    None: data_default,
    'bool': data_bool,
    'cat': data_cat,
    'sparse': data_sparse

}

def get_spec_no_mutate(entry, backend):
    return "no_mutate" in entry['backends'].get(backend, {}).get('flags', [])

def get_spec_backend_status(entry, backend):
    return entry['backends'].get(backend, {}).get('status')

def get_spec_sql_type(entry, backend):
    return entry['backends'].get(backend, {}).get('result_type')

def get_data(entry, data, backend = None):

    req_bool = entry['action'].get('input_type') == 'bool'

    # pandas is forgiving to bool inputs
    if isinstance(backend, PandasBackend):
        req_bool = False

    return data['bool'] if req_bool else data[entry['accessor']]

def test_missing_implementation(entry, backend):
    # Check whether test should xfail, skip, or -------------------------------
    backend_status = get_spec_backend_status(entry, backend.name)

    # case: Needs to be implmented
    # TODO(table): uses xfail
    if backend_status == "todo":
        pytest.xfail("TODO: impelement this translation")

    # case: Can't be used in a mutate (e.g. a SQL ordered set aggregate function)
    # TODO(table): no_mutate
    if get_spec_no_mutate(entry, backend.name):
        pytest.skip("Spec'd failure")

    # case: won't be implemented
    if get_spec_backend_status(entry, backend.name) == "wontdo":
        pytest.skip()

def get_df_expr(entry):
    str_expr = str(entry['expr_frame'])
    call_expr = strip_symbolic(eval(str_expr, {'_': _}))

    return str_expr, call_expr

def cast_result_type(entry, backend, ser):
    sql_type = get_spec_sql_type(entry, backend.name)
    if isinstance(backend, SqlBackend) and sql_type == 'float':
        return ser.astype('float')

    return ser

# Tests =======================================================================

def test_series_against_call(entry):
    if entry['action']['kind'] == "window":
        pytest.skip()

    df = data[entry['accessor']]
    # TODO: once reading from yaml, no need to repr
    str_expr = str(entry['expr_series'])

    call_expr = strip_symbolic(eval(str_expr, {'_': _}))
    res = call_expr(df.x)

    dst = eval(str_expr, {'_': df.x})

    assert res.__class__ is dst.__class__
    assert_src_array_equal(res, dst)

def test_frame_expr(entry):
    # TODO: remove this test, not checking anything new
    df = data[entry['accessor']]
    # TODO: once reading from yaml, no need to repr
    str_expr = str(entry['expr_frame'])

    call_expr = strip_symbolic(eval(str_expr, {'_': _}))
    res = call_expr(df)

    dst = eval(str_expr, {'_': df})

    assert res.__class__ is dst.__class__
    assert_src_array_equal(res, dst)

def test_pandas_grouped_frame_fast_not_implemented(notimpl_entry):
    from siuba.experimental.pd_groups.dialect import fast_mutate
    gdf = data[notimpl_entry['accessor']].groupby('g')

    # TODO: once reading from yaml, no need to repr
    str_expr = str(notimpl_entry['expr_frame'])
    call_expr = strip_symbolic(eval(str_expr, {'_': _}))

    if notimpl_entry['action']['status'] in ["todo", "maydo", "wontdo"] and notimpl_entry["is_property"]:
        pytest.xfail()

    with pytest.raises(NotImplementedError):
        res = fast_mutate(gdf, result = call_expr)

#@backend_pandas
@pytest.mark.skip_backend('sqlite')
def test_frame_mutate(skip_backend, backend, entry):
    test_missing_implementation(entry, backend)

    # Prepare input data ------------------------------------------------------
    # case: inputs must be boolean
    crnt_data = get_data(entry, DATA, backend)
    df = backend.load_df(crnt_data)

    # Execute mutate ----------------------------------------------------------
    str_expr, call_expr = get_df_expr(entry)

    # Run test for equality w/ ungrouped pandas ----
    dst = crnt_data.assign(result = call_expr(crnt_data))
    dst['result'] = cast_result_type(entry, backend, dst['result'])

    assert_equal_query(
            df,
            mutate(result = call_expr),
            dst
            )

    # Run test for equality w/ grouped pandas ----
    g_dst = crnt_data.groupby('g').apply(lambda d: d.assign(result = call_expr)).reset_index(drop = True)
    g_dst['result'] = cast_result_type(entry, backend, g_dst['result'])
    assert_equal_query(
            df,
            group_by(_.g) >> mutate(result = call_expr),
            g_dst
            )

def test_pandas_grouped_frame_fast_mutate(entry):
    from siuba.experimental.pd_groups.dialect import fast_mutate, DataFrameGroupBy
    gdf = get_data(entry, DATA).groupby('g')

    # Execute mutate ----------------------------------------------------------
    str_expr, call_expr = get_df_expr(entry)

    res = fast_mutate(gdf, result = call_expr)
    dst = mutate(gdf, result = call_expr)

    # TODO: apply mark to skip failing tests, rather than downcast
    # pandas grouped aggs, when not using cython, _try_cast back to original type
    # but since mutate uses apply, it doesn't :/. Currently only affects median func.
    dst_obj = dst.obj
    if str_expr == '_.x.median()':
        dst_obj['result'] = dst_obj['result'].astype(gdf.x.obj.dtype)

    assert isinstance(dst, DataFrameGroupBy)
    assert_frame_equal(res.obj, dst_obj)

@pytest.mark.skip_backend('sqlite')
def test_frame_summarize(skip_backend, backend, agg_entry):
    entry = agg_entry
    test_missing_implementation(entry, backend)

    # Prepare input data ------------------------------------------------------
    # case: inputs must be boolean
    crnt_data = get_data(entry, DATA, backend)
    df = backend.load_df(crnt_data)

    # Execute mutate ----------------------------------------------------------
    str_expr, call_expr = get_df_expr(entry)

    dst = data_frame(result = call_expr(crnt_data))

    # Process output ----------------------------------------------------------
    # case: output is of a different type than w/ pandas
    dst['result'] = cast_result_type(entry, backend, dst['result'])

    # Run test for equality w/ pandas ----
    # otherwise, verify returns same result as mutate
    assert_equal_query(
            df,
            summarize(result = call_expr),
            dst
            )

    dst_g = crnt_data.groupby('g').apply(call_expr).reset_index().rename(columns = {0: 'result'})
    assert_equal_query(
            df,
            group_by(_.g) >> summarize(result = call_expr),
            dst_g
            )

def test_pandas_grouped_frame_fast_summarize(agg_entry):
    from siuba.experimental.pd_groups.dialect import fast_summarize, DataFrameGroupBy
    gdf = get_data(agg_entry, DATA).groupby('g')

    # Execute summarize ----------------------------------------------------------
    str_expr, call_expr = get_df_expr(agg_entry)

    res = fast_summarize(gdf, result = call_expr)
    dst = summarize(gdf, result = call_expr)

    # TODO: apply mark to skip failing tests, rather than downcast
    # pandas grouped aggs, when not using cython, _try_cast back to original type
    # but since summarize uses apply, it doesn't :/. Currently only affects median func.
    if str_expr == '_.x.median()':
        dst['result'] = dst['result'].astype(gdf.x.obj.dtype)

    assert_frame_equal(res, dst)

# Edge Cases ==================================================================

def test_frame_set_aggregates_postgresql():
    # TODO: probably shouldn't be creating backend here
    backend = SqlBackend("postgresql")
    dfs = backend.load_df(data[None])

    expr = _.x.quantile(.75)
    assert_equal_query(
            dfs,
            group_by(_.g) >> summarize(result = expr),
            data_frame(g = ['a', 'b'], result = [11., 13.])
            )

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from django.contrib import admin
from djangotrellostats.apps.members.models import Member

admin.site.register(Member)<commit_msg>End of file \n<commit_after># -*- coding: utf-8 -*-

from django.contrib import admin
from djangotrellostats.apps.members.models import Member

admin.site.register(Member)
<|endoftext|>


<commit_before>#
# ovirt-engine-setup -- ovirt engine setup
# Copyright (C) 2013-2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import gettext
import re
import socket

from otopi import base, util

from ovirt_engine_setup import constants as osetupcons
from ovirt_engine_setup import dialog

def _(m):
    return gettext.dgettext(message=m, domain='ovirt-engine-setup')

@util.export
class Hostname(base.Base):
    _IPADDR_RE = re.compile(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}')

    _DOMAIN_RE = re.compile(
        flags=re.VERBOSE,
        pattern=r"""
            ^
            [A-Za-z0-9\.\-]+
            \w+
            $
        """
    )

    _INTERFACE_RE = re.compile(
        flags=re.VERBOSE,
        pattern=r"""
            ^
            \d+
            :
            \s+
            (?P<interface>\w+(\.\w+)?)(@\w+)?
            :
            \s+
            <(?P<options>[^>]+)
            .*
        """
    )

    _ADDRESS_RE = re.compile(
        flags=re.VERBOSE,
        pattern=r"""
            \s+
            inet
            \s
            (?P<address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})
            .+
            \s+
            (?P<interface>\w+(\.\w+(@\w+)?)?)
            $
    """
    )

    _DIG_LOOKUP_RE = re.compile(
        flags=re.VERBOSE,
        pattern=r"""
            ^
            [\w.-]+
            \s+
            \d+
            \s+
            IN
            \s+
            (A|CNAME)
            \s+
            [\w.-]+
        """
    )

    _DIG_REVLOOKUP_RE = re.compile(
        flags=re.VERBOSE,
        pattern=r"""
            ^
            [\w/.-]+\.in-addr\.arpa\.
            \s+
            \d+
            \s+
            IN
            \s+
            PTR
            \s+
            (?P<answer>[\w/.-]+)
            \.
            $
        """
    )

    def __init__(self, plugin):
        super(Hostname, self).__init__()
        self._plugin = plugin
        self.command.detect('dig')

    @property
    def plugin(self):
        return self._plugin

    @property
    def command(self):
        return self._plugin.command

    @property
    def dialog(self):
        return self._plugin.dialog

    @property
    def execute(self):
        return self._plugin.execute

    @property
    def environment(self):
        return self._plugin.environment

    @property
    def logger(self):
        return self._plugin.logger

    def _getNonLoopbackAddresses(self):
        interfaces = {}
        addresses = {}
        rc, stdout, stderr = self.execute(
            args=(
                self.command.get('ip'),
                'addr',
            ),
        )
        for line in stdout:
            interfacematch = self._INTERFACE_RE.match(line)
            addressmatch = self._ADDRESS_RE.match(line)
            if interfacematch is not None:
                interfaces[
                    interfacematch.group('interface')
                ] = 'LOOPBACK' in interfacematch.group('options')
            elif addressmatch is not None:
                addresses.setdefault(
                    addressmatch.group('interface'),
                    []
                ).append(
                    addressmatch.group('address')
                )
        iplist = []
        for interface, loopback in interfaces.iteritems():
            if not loopback:
                iplist.extend(addresses.get(interface, []))

        self.logger.debug('addresses: %s' % iplist)
        return set(iplist)

    def _dig_reverse_lookup(self, addr):
        names = set()
        args = [
            self.command.get('dig'),
            '-x',
            addr,
        ]
        rc, stdout, stderr = self.execute(
            args=args,
            raiseOnError=False
        )
        if rc == 0:
            for line in stdout:
                found = self._DIG_REVLOOKUP_RE.search(line)
                if found:
                    names.add(found.group('answer'))
        return names

    def _validateFQDNresolvability(self, fqdn):

        try:
            resolvedAddresses = self.getResolvedAddresses(fqdn)
            self.logger.debug(
                '{fqdn} resolves to: {addresses}'.format(
                    fqdn=fqdn,
                    addresses=resolvedAddresses,
                )
            )
            resolvedAddressesAsString = ' '.join(resolvedAddresses)
        except socket.error:
            raise RuntimeError(
                _('{fqdn} did not resolve into an IP address').format(
                    fqdn=fqdn,
                )
            )

        resolvedByDNS = self.isResolvedByDNS(fqdn)
        if not resolvedByDNS:
            self.logger.warning(
                _(
                    'Failed to resolve {fqdn} using DNS, '
                    'it can be resolved only locally'
                ).format(
                    fqdn=fqdn,
                )
            )
        elif self.environment[osetupcons.ConfigEnv.FQDN_REVERSE_VALIDATION]:
            revResolved = False
            for address in resolvedAddresses:
                for name in self._dig_reverse_lookup(address):
                    revResolved = name.lower() == fqdn.lower()
                    if revResolved:
                        break
                if revResolved:
                    break
            if not revResolved:
                raise RuntimeError(
                    _(
                        'The following addresses: {addresses} did not reverse '
                        'resolve into {fqdn}'
                    ).format(
                        addresses=resolvedAddressesAsString,
                        fqdn=fqdn
                    )
                )

        if self.environment[
            osetupcons.ConfigEnv.FQDN_NON_LOOPBACK_VALIDATION
        ]:
            if not resolvedAddresses.issubset(self._getNonLoopbackAddresses()):
                raise RuntimeError(
                    _(
                        'The following addreses: '
                        "{addresses} can't be mapped "
                        'to non loopback devices on this host'
                    ).format(
                        addresses=resolvedAddressesAsString
                    )
                )

    def _validateFQDN(self, fqdn):
        if self._IPADDR_RE.match(fqdn):
            raise RuntimeError(
                _(
                    '{fqdn} is an IP address and not a FQDN. '
                    'A FQDN is needed to be able to generate '
                    'certificates correctly.'
                ).format(
                    fqdn=fqdn,
                )
            )

        if not fqdn:
            raise RuntimeError(
                _('Please specify host FQDN')
            )

        if len(fqdn) > 1000:
            raise RuntimeError(
                _('FQDN has invalid length')
            )

        components = fqdn.split('.', 1)
        if len(components) == 1 or not components[0]:
            self.logger.warning(
                _('Host name {fqdn} has no domain suffix').format(
                    fqdn=fqdn,
                )
            )
        else:
            if not self._DOMAIN_RE.match(components[1]):
                raise RuntimeError(
                    _('Host name {fqdn} has invalid domain name').format(
                        fqdn=fqdn,
                    )
                )

    def isResolvedByDNS(self, fqdn):
        args = [
            self.command.get('dig'),
            fqdn
        ]
        rc, stdout, stderr = self.execute(
            args=args,
            raiseOnError=False
        )
        resolved = False
        if rc == 0:
            for line in stdout:
                if self._DIG_LOOKUP_RE.search(line):
                    resolved = True
        return resolved

    def getResolvedAddresses(self, fqdn):
        return set([
            address[0] for __, __, __, __, address in
            socket.getaddrinfo(
                fqdn,
                None,
                # Currently we need an IPv4 address and ignore the rest.
                socket.AF_INET,
            )
        ])

    def getHostname(self, envkey, whichhost, supply_default, prompttext=None):

        def test_hostname(name):
            res = ''
            try:
                self._validateFQDN(name)
                self._validateFQDNresolvability(name)
            except RuntimeError as e:
                res = _('Host name is not valid: {e}').format(e=e)
                self.logger.debug('test_hostname exception', exc_info=True)
            return res

        if prompttext is None:
            prompttext = _(
                'Host fully qualified DNS name of {whichhost} server'
            ).format(
                whichhost=whichhost,
            )
        return dialog.queryEnvKey(
            name='OVESETUP_NETWORK_FQDN_{whichhost}'.format(
                whichhost=whichhost.replace(' ', '_'),
            ),
            dialog=self.dialog,
            logger=self.logger,
            env=self.environment,
            key=envkey,
            note=_(
                '{prompt} [@DEFAULT@]: '
            ).format(
                prompt=prompttext,
            ),
            prompt=True,
            default=socket.getfqdn() if supply_default else '',
            tests=(
                {
                    'test': test_hostname,
                },
            ),
        )

# vim: expandtab tabstop=4 shiftwidth=4
<commit_msg>packaging: setup: validate full fqdn<commit_after>#
# ovirt-engine-setup -- ovirt engine setup
# Copyright (C) 2013-2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import gettext
import re
import socket

from otopi import base, util

from ovirt_engine_setup import constants as osetupcons
from ovirt_engine_setup import dialog

def _(m):
    return gettext.dgettext(message=m, domain='ovirt-engine-setup')

@util.export
class Hostname(base.Base):
    _IPADDR_RE = re.compile(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}')

    _DOMAIN_RE = re.compile(
        flags=re.VERBOSE,
        pattern=r"""
            ^
            [A-Za-z0-9\.\-]+
            \w+
            $
        """
    )

    _INTERFACE_RE = re.compile(
        flags=re.VERBOSE,
        pattern=r"""
            ^
            \d+
            :
            \s+
            (?P<interface>\w+(\.\w+)?)(@\w+)?
            :
            \s+
            <(?P<options>[^>]+)
            .*
        """
    )

    _ADDRESS_RE = re.compile(
        flags=re.VERBOSE,
        pattern=r"""
            \s+
            inet
            \s
            (?P<address>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})
            .+
            \s+
            (?P<interface>\w+(\.\w+(@\w+)?)?)
            $
    """
    )

    _DIG_LOOKUP_RE = re.compile(
        flags=re.VERBOSE,
        pattern=r"""
            ^
            [\w.-]+
            \s+
            \d+
            \s+
            IN
            \s+
            (A|CNAME)
            \s+
            [\w.-]+
        """
    )

    _DIG_REVLOOKUP_RE = re.compile(
        flags=re.VERBOSE,
        pattern=r"""
            ^
            [\w/.-]+\.in-addr\.arpa\.
            \s+
            \d+
            \s+
            IN
            \s+
            PTR
            \s+
            (?P<answer>[\w/.-]+)
            \.
            $
        """
    )

    def __init__(self, plugin):
        super(Hostname, self).__init__()
        self._plugin = plugin
        self.command.detect('dig')

    @property
    def plugin(self):
        return self._plugin

    @property
    def command(self):
        return self._plugin.command

    @property
    def dialog(self):
        return self._plugin.dialog

    @property
    def execute(self):
        return self._plugin.execute

    @property
    def environment(self):
        return self._plugin.environment

    @property
    def logger(self):
        return self._plugin.logger

    def _getNonLoopbackAddresses(self):
        interfaces = {}
        addresses = {}
        rc, stdout, stderr = self.execute(
            args=(
                self.command.get('ip'),
                'addr',
            ),
        )
        for line in stdout:
            interfacematch = self._INTERFACE_RE.match(line)
            addressmatch = self._ADDRESS_RE.match(line)
            if interfacematch is not None:
                interfaces[
                    interfacematch.group('interface')
                ] = 'LOOPBACK' in interfacematch.group('options')
            elif addressmatch is not None:
                addresses.setdefault(
                    addressmatch.group('interface'),
                    []
                ).append(
                    addressmatch.group('address')
                )
        iplist = []
        for interface, loopback in interfaces.iteritems():
            if not loopback:
                iplist.extend(addresses.get(interface, []))

        self.logger.debug('addresses: %s' % iplist)
        return set(iplist)

    def _dig_reverse_lookup(self, addr):
        names = set()
        args = [
            self.command.get('dig'),
            '-x',
            addr,
        ]
        rc, stdout, stderr = self.execute(
            args=args,
            raiseOnError=False
        )
        if rc == 0:
            for line in stdout:
                found = self._DIG_REVLOOKUP_RE.search(line)
                if found:
                    names.add(found.group('answer'))
        return names

    def _validateFQDNresolvability(self, fqdn):

        try:
            resolvedAddresses = self.getResolvedAddresses(fqdn)
            self.logger.debug(
                '{fqdn} resolves to: {addresses}'.format(
                    fqdn=fqdn,
                    addresses=resolvedAddresses,
                )
            )
            resolvedAddressesAsString = ' '.join(resolvedAddresses)
        except socket.error:
            raise RuntimeError(
                _('{fqdn} did not resolve into an IP address').format(
                    fqdn=fqdn,
                )
            )

        resolvedByDNS = self.isResolvedByDNS(fqdn)
        if not resolvedByDNS:
            self.logger.warning(
                _(
                    'Failed to resolve {fqdn} using DNS, '
                    'it can be resolved only locally'
                ).format(
                    fqdn=fqdn,
                )
            )
        elif self.environment[osetupcons.ConfigEnv.FQDN_REVERSE_VALIDATION]:
            revResolved = False
            for address in resolvedAddresses:
                for name in self._dig_reverse_lookup(address):
                    revResolved = name.lower() == fqdn.lower()
                    if revResolved:
                        break
                if revResolved:
                    break
            if not revResolved:
                raise RuntimeError(
                    _(
                        'The following addresses: {addresses} did not reverse '
                        'resolve into {fqdn}'
                    ).format(
                        addresses=resolvedAddressesAsString,
                        fqdn=fqdn
                    )
                )

        if self.environment[
            osetupcons.ConfigEnv.FQDN_NON_LOOPBACK_VALIDATION
        ]:
            if not resolvedAddresses.issubset(self._getNonLoopbackAddresses()):
                raise RuntimeError(
                    _(
                        'The following addreses: '
                        "{addresses} can't be mapped "
                        'to non loopback devices on this host'
                    ).format(
                        addresses=resolvedAddressesAsString
                    )
                )

    def _validateFQDN(self, fqdn):
        if self._IPADDR_RE.match(fqdn):
            raise RuntimeError(
                _(
                    '{fqdn} is an IP address and not a FQDN. '
                    'A FQDN is needed to be able to generate '
                    'certificates correctly.'
                ).format(
                    fqdn=fqdn,
                )
            )

        if not fqdn:
            raise RuntimeError(
                _('Please specify host FQDN')
            )

        if len(fqdn) > 1000:
            raise RuntimeError(
                _('FQDN has invalid length')
            )

        components = fqdn.split('.', 1)
        if len(components) == 1 or not components[0]:
            self.logger.warning(
                _('Host name {fqdn} has no domain suffix').format(
                    fqdn=fqdn,
                )
            )
        else:
            if not self._DOMAIN_RE.match(fqdn):
                raise RuntimeError(
                    _('Host name {fqdn} is not valid').format(
                        fqdn=fqdn,
                    )
                )

    def isResolvedByDNS(self, fqdn):
        args = [
            self.command.get('dig'),
            fqdn
        ]
        rc, stdout, stderr = self.execute(
            args=args,
            raiseOnError=False
        )
        resolved = False
        if rc == 0:
            for line in stdout:
                if self._DIG_LOOKUP_RE.search(line):
                    resolved = True
        return resolved

    def getResolvedAddresses(self, fqdn):
        return set([
            address[0] for __, __, __, __, address in
            socket.getaddrinfo(
                fqdn,
                None,
                # Currently we need an IPv4 address and ignore the rest.
                socket.AF_INET,
            )
        ])

    def getHostname(self, envkey, whichhost, supply_default, prompttext=None):

        def test_hostname(name):
            res = ''
            try:
                self._validateFQDN(name)
                self._validateFQDNresolvability(name)
            except RuntimeError as e:
                res = _('Host name is not valid: {e}').format(e=e)
                self.logger.debug('test_hostname exception', exc_info=True)
            return res

        if prompttext is None:
            prompttext = _(
                'Host fully qualified DNS name of {whichhost} server'
            ).format(
                whichhost=whichhost,
            )
        return dialog.queryEnvKey(
            name='OVESETUP_NETWORK_FQDN_{whichhost}'.format(
                whichhost=whichhost.replace(' ', '_'),
            ),
            dialog=self.dialog,
            logger=self.logger,
            env=self.environment,
            key=envkey,
            note=_(
                '{prompt} [@DEFAULT@]: '
            ).format(
                prompt=prompttext,
            ),
            prompt=True,
            default=socket.getfqdn() if supply_default else '',
            tests=(
                {
                    'test': test_hostname,
                },
            ),
        )

# vim: expandtab tabstop=4 shiftwidth=4
<|endoftext|>


<commit_before>{
	'repo_type' : 'archive',
	'download_locations' : [
		{ 'url' : 'https://www.gnupg.org/ftp/gcrypt/gnutls/v3.6/gnutls-3.6.10.tar.xz', 'hashes' : [ { 'type' : 'sha256', 'sum' : 'b1f3ca67673b05b746a961acf2243eaae0ffe658b6a6494265c648e7c7812293' }, ], },
		{ 'url' : 'https://fossies.org/linux/misc/gnutls-3.6.10.tar.xz', 'hashes' : [ { 'type' : 'sha256', 'sum' : 'b1f3ca67673b05b746a961acf2243eaae0ffe658b6a6494265c648e7c7812293' }, ], },
	],
	'configure_options' :
		'--host={target_host} --prefix={target_prefix} --disable-shared --enable-static '
		'--disable-srp-authentication '
		'--disable-non-suiteb-curves '
		'--enable-cxx '
		'--enable-nls '
		'--disable-rpath '
		'--disable-gtk-doc '
		'--disable-guile '
		'--disable-doc '
		'--enable-local-libopts '
		'--disable-tools '
		'--disable-tests '
		'--with-zlib '
		'--with-included-libtasn1 '
		'--with-included-unistring '
		'--with-default-trust-store-file '
		'--with-default-blacklist-file '
		'--without-tpm '
		'--without-p11-kit'
	,
	'regex_replace': {
		'post_install': [
			{
				0: r'^(Libs: -L\${{libdir}} -lgnutls)([\n\r\s]+)?$',
				1: r'\1 -lnettle -lhogweed -lgmp -lcrypt32 -lws2_32 -lintl -liconv\2', # iconv is required by gettext, but gettext has no .pc file, so...
				'in_file': '{pkg_config_path}/gnutls.pc'
			},
		],
	},
	# 'patches' : [
		#('gnutls/rename-inet_pton_for_srt.diff', '-p1'),
		# ('https://raw.githubusercontent.com/DeadSix27/python_cross_compile_script/master/patches/gnutls/0001-gnutls-3.5.11-arpainet_pkgconfig.patch', '-p1'),
		# ('https://raw.githubusercontent.com/Martchus/PKGBUILDs/master/gnutls/mingw-w64/gnutls-3.2.7-rpath.patch','-p1'),
		# ('https://raw.githubusercontent.com/Martchus/PKGBUILDs/master/gnutls/mingw-w64/gnutls-fix-external-libtasn1-detection.patch','-p1'),
	# ],
	'depends_on' : [
		'gettext',
		'iconv',
		'libnettle',
		'zlib',
	],
	'update_check' : { 'url' : 'ftp://ftp.gnutls.org/gcrypt/gnutls/v3.6', 'type' : 'ftpindex', 'regex' : r'gnutls-(?P<version_num>[\d.]+)\.tar\.xz' },
	'_info' : { 'version' : '3.6.10', 'fancy_name' : 'gnutls' },
}<commit_msg>packages/gnutls: update to 3.6.11.1<commit_after>{
	'repo_type' : 'archive',
	'download_locations' : [
		{ 'url' : 'https://www.gnupg.org/ftp/gcrypt/gnutls/v3.6/gnutls-3.6.11.1.tar.xz', 'hashes' : [ { 'type' : 'sha256', 'sum' : 'fbba12f3db9a55dbf027e14111755817ec44b57eabec3e8089aac8ac6f533cf8' }, ], },
		{ 'url' : 'https://fossies.org/linux/misc/gnutls-3.6.11.1.tar.xz', 'hashes' : [ { 'type' : 'sha256', 'sum' : 'fbba12f3db9a55dbf027e14111755817ec44b57eabec3e8089aac8ac6f533cf8' }, ], },
	],
	'configure_options' :
		'--host={target_host} --prefix={target_prefix} --disable-shared --enable-static '
		'--disable-srp-authentication '
		'--disable-non-suiteb-curves '
		'--enable-cxx '
		'--enable-nls '
		'--disable-rpath '
		'--disable-gtk-doc '
		'--disable-guile '
		'--disable-doc '
		'--enable-local-libopts '
		'--disable-tools '
		'--disable-tests '
		'--with-zlib '
		'--with-included-libtasn1 '
		'--with-included-unistring '
		'--with-default-trust-store-file '
		'--with-default-blacklist-file '
		'--without-tpm '
		'--without-p11-kit'
	,
	'regex_replace': {
		'post_install': [
			{
				0: r'^(Libs: -L\${{libdir}} -lgnutls)([\n\r\s]+)?$',
				1: r'\1 -lnettle -lhogweed -lgmp -lcrypt32 -lws2_32 -lintl -liconv\2', # iconv is required by gettext, but gettext has no .pc file, so...
				'in_file': '{pkg_config_path}/gnutls.pc'
			},
		],
	},
	# 'patches' : [
		#('gnutls/rename-inet_pton_for_srt.diff', '-p1'),
		# ('https://raw.githubusercontent.com/DeadSix27/python_cross_compile_script/master/patches/gnutls/0001-gnutls-3.5.11-arpainet_pkgconfig.patch', '-p1'),
		# ('https://raw.githubusercontent.com/Martchus/PKGBUILDs/master/gnutls/mingw-w64/gnutls-3.2.7-rpath.patch','-p1'),
		# ('https://raw.githubusercontent.com/Martchus/PKGBUILDs/master/gnutls/mingw-w64/gnutls-fix-external-libtasn1-detection.patch','-p1'),
	# ],
	'depends_on' : [
		'gettext',
		'iconv',
		'libnettle',
		'zlib',
	],
	'update_check' : { 'url' : 'ftp://ftp.gnutls.org/gcrypt/gnutls/v3.6', 'type' : 'ftpindex', 'regex' : r'gnutls-(?P<version_num>[\d.]+)\.tar\.xz' },
	'_info' : { 'version' : '3.6.11.1', 'fancy_name' : 'gnutls' },
}<|endoftext|>