<commit_before>import asyncio
from functools import partial
from inspect import isawaitable
from signal import SIGINT, SIGTERM
from time import time

import httptools

try:
    import uvloop as async_loop
except ImportError:
    async_loop = asyncio

from .log import log
from .request import Request

class Signal:
    stopped = False

current_time = None

class HttpProtocol(asyncio.Protocol):
    __slots__ = (
        # event loop, connection
        'loop', 'transport', 'connections', 'signal',
        # request params
        'parser', 'request', 'url', 'headers',
        # request config
        'request_handler', 'request_timeout', 'request_max_size',
        # connection management
        '_total_request_size', '_timeout_handler', '_last_communication_time')

    def __init__(self, *, loop, request_handler, signal=Signal(),
                 connections={}, request_timeout=60,
                 request_max_size=None):
        self.loop = loop
        self.transport = None
        self.request = None
        self.parser = None
        self.url = None
        self.headers = None
        self.signal = signal
        self.connections = connections
        self.request_handler = request_handler
        self.request_timeout = request_timeout
        self.request_max_size = request_max_size
        self._total_request_size = 0
        self._timeout_handler = None
        self._last_request_time = None

    # -------------------------------------------- #
    # Connection
    # -------------------------------------------- #

    def connection_made(self, transport):
        self.connections[self] = True
        self._timeout_handler = self.loop.call_later(
            self.request_timeout, self.connection_timeout)
        self.transport = transport
        self._last_request_time = current_time

    def connection_lost(self, exc):
        del self.connections[self]
        self._timeout_handler.cancel()
        self.cleanup()

    def connection_timeout(self):
        # Check if
        time_elapsed = current_time - self._last_request_time
        if time_elapsed < self.request_timeout:
            time_left = self.request_timeout - time_elapsed
            self._timeout_handler = \
                self.loop.call_later(time_left, self.connection_timeout)
        else:
            self.bail_out("Request timed out, connection closed")

    # -------------------------------------------- #
    # Parsing
    # -------------------------------------------- #

    def data_received(self, data):
        # Check for the request itself getting too large and exceeding
        # memory limits
        self._total_request_size += len(data)
        if self._total_request_size > self.request_max_size:
            return self.bail_out(
                "Request too large ({}), connection closed".format(
                    self._total_request_size))

        # Create parser if this is the first time we're receiving data
        if self.parser is None:
            assert self.request is None
            self.headers = []
            self.parser = httptools.HttpRequestParser(self)

        # Parse request chunk or close connection
        try:
            self.parser.feed_data(data)
        except httptools.parser.errors.HttpParserError as e:
            self.bail_out(
                "Invalid request data, connection closed ({})".format(e))

    def on_url(self, url):
        self.url = url

    def on_header(self, name, value):
        if name == b'Content-Length' and int(value) > self.request_max_size:
            return self.bail_out(
                "Request body too large ({}), connection closed".format(value))

        self.headers.append((name.decode(), value.decode('utf-8')))

    def on_headers_complete(self):
        self.request = Request(
            url_bytes=self.url,
            headers=dict(self.headers),
            version=self.parser.get_http_version(),
            method=self.parser.get_method().decode()
        )

    def on_body(self, body):
        if self.request.body:
            self.request.body += body
        else:
            self.request.body = body

    def on_message_complete(self):
        self.loop.create_task(
            self.request_handler(self.request, self.write_response))

    # -------------------------------------------- #
    # Responding
    # -------------------------------------------- #

    def write_response(self, response):
        try:
            keep_alive = self.parser.should_keep_alive() \
                            and not self.signal.stopped
            self.transport.write(
                response.output(
                    self.request.version, keep_alive, self.request_timeout))
            if not keep_alive:
                self.transport.close()
            else:
                # Record that we received data
                self._last_request_time = current_time
                self.cleanup()
        except Exception as e:
            self.bail_out(
                "Writing request failed, connection closed {}".format(e))

    def bail_out(self, message):
        log.debug(message)
        self.transport.close()

    def cleanup(self):
        self.parser = None
        self.request = None
        self.url = None
        self.headers = None
        self._total_request_size = 0

    def close_if_idle(self):
        """
        Close the connection if a request is not being sent or received
        :return: boolean - True if closed, false if staying open
        """
        if not self.parser:
            self.transport.close()
            return True
        return False

def update_current_time(loop):
    """
    Caches the current time, since it is needed
    at the end of every keep-alive request to update the request timeout time
    :param loop:
    :return:
    """
    global current_time
    current_time = time()
    loop.call_later(1, partial(update_current_time, loop))

def trigger_events(events, loop):
    """
    :param events: one or more sync or async functions to execute
    :param loop: event loop
    """
    if events:
        if not isinstance(events, list):
            events = [events]
        for event in events:
            result = event(loop)
            if isawaitable(result):
                loop.run_until_complete(result)

def serve(host, port, request_handler, before_start=None, after_start=None,
          before_stop=None, after_stop=None,
          debug=False, request_timeout=60, sock=None,
          request_max_size=None, reuse_port=False, loop=None):
    """
    Starts asynchronous HTTP Server on an individual process.
    :param host: Address to host on
    :param port: Port to host on
    :param request_handler: Sanic request handler with middleware
    :param after_start: Function to be executed after the server starts
    listening. Takes single argument `loop`
    :param before_stop: Function to be executed when a stop signal is
    received before it is respected. Takes single argumenet `loop`
    :param debug: Enables debug output (slows server)
    :param request_timeout: time in seconds
    :param sock: Socket for the server to accept connections from
    :param request_max_size: size in bytes, `None` for no limit
    :param reuse_port: `True` for multiple workers
    :param loop: asyncio compatible event loop
    :return: Nothing
    """
    loop = loop or async_loop.new_event_loop()
    asyncio.set_event_loop(loop)

    if debug:
        loop.set_debug(debug)

    trigger_events(before_start, loop)

    connections = {}
    signal = Signal()
    server_coroutine = loop.create_server(lambda: HttpProtocol(
        loop=loop,
        connections=connections,
        signal=signal,
        request_handler=request_handler,
        request_timeout=request_timeout,
        request_max_size=request_max_size,
    ), host, port, reuse_port=reuse_port, sock=sock)

    # Instead of pulling time at the end of every request,
    # pull it once per minute
    loop.call_soon(partial(update_current_time, loop))

    try:
        http_server = loop.run_until_complete(server_coroutine)
    except Exception:
        log.exception("Unable to start server")
        return

    trigger_events(after_start, loop)

    # Register signals for graceful termination
    for _signal in (SIGINT, SIGTERM):
        loop.add_signal_handler(_signal, loop.stop)

    try:
        loop.run_forever()
    finally:
        log.info("Stop requested, draining connections...")

        # Run the on_stop function if provided
        trigger_events(before_stop, loop)

        # Wait for event loop to finish and all connections to drain
        http_server.close()
        loop.run_until_complete(http_server.wait_closed())

        # Complete all tasks on the loop
        signal.stopped = True
        for connection in connections.keys():
            connection.close_if_idle()

        while connections:
            loop.run_until_complete(asyncio.sleep(0.1))

        trigger_events(after_stop, loop)

        loop.close()
<commit_msg>Add the client address to the request header<commit_after>import asyncio
from functools import partial
from inspect import isawaitable
from signal import SIGINT, SIGTERM
from time import time

import httptools

try:
    import uvloop as async_loop
except ImportError:
    async_loop = asyncio

from .log import log
from .request import Request

class Signal:
    stopped = False

current_time = None

class HttpProtocol(asyncio.Protocol):
    __slots__ = (
        # event loop, connection
        'loop', 'transport', 'connections', 'signal',
        # request params
        'parser', 'request', 'url', 'headers',
        # request config
        'request_handler', 'request_timeout', 'request_max_size',
        # connection management
        '_total_request_size', '_timeout_handler', '_last_communication_time')

    def __init__(self, *, loop, request_handler, signal=Signal(),
                 connections={}, request_timeout=60,
                 request_max_size=None):
        self.loop = loop
        self.transport = None
        self.request = None
        self.parser = None
        self.url = None
        self.headers = None
        self.signal = signal
        self.connections = connections
        self.request_handler = request_handler
        self.request_timeout = request_timeout
        self.request_max_size = request_max_size
        self._total_request_size = 0
        self._timeout_handler = None
        self._last_request_time = None

    # -------------------------------------------- #
    # Connection
    # -------------------------------------------- #

    def connection_made(self, transport):
        self.connections[self] = True
        self._timeout_handler = self.loop.call_later(
            self.request_timeout, self.connection_timeout)
        self.transport = transport
        self._last_request_time = current_time

    def connection_lost(self, exc):
        del self.connections[self]
        self._timeout_handler.cancel()
        self.cleanup()

    def connection_timeout(self):
        # Check if
        time_elapsed = current_time - self._last_request_time
        if time_elapsed < self.request_timeout:
            time_left = self.request_timeout - time_elapsed
            self._timeout_handler = \
                self.loop.call_later(time_left, self.connection_timeout)
        else:
            self.bail_out("Request timed out, connection closed")

    # -------------------------------------------- #
    # Parsing
    # -------------------------------------------- #

    def data_received(self, data):
        # Check for the request itself getting too large and exceeding
        # memory limits
        self._total_request_size += len(data)
        if self._total_request_size > self.request_max_size:
            return self.bail_out(
                "Request too large ({}), connection closed".format(
                    self._total_request_size))

        # Create parser if this is the first time we're receiving data
        if self.parser is None:
            assert self.request is None
            self.headers = []
            self.parser = httptools.HttpRequestParser(self)

        # Parse request chunk or close connection
        try:
            self.parser.feed_data(data)
        except httptools.parser.errors.HttpParserError as e:
            self.bail_out(
                "Invalid request data, connection closed ({})".format(e))

    def on_url(self, url):
        self.url = url

    def on_header(self, name, value):
        if name == b'Content-Length' and int(value) > self.request_max_size:
            return self.bail_out(
                "Request body too large ({}), connection closed".format(value))

        self.headers.append((name.decode(), value.decode('utf-8')))

    def on_headers_complete(self):

        ra = self.transport.get_extra_info('peername')
        if ra:
             self.headers.append(('Remote-Addr','%s:%s' % ra))

        self.request = Request(
            url_bytes=self.url,
            headers=dict(self.headers),
            version=self.parser.get_http_version(),
            method=self.parser.get_method().decode()
        )

    def on_body(self, body):
        if self.request.body:
            self.request.body += body
        else:
            self.request.body = body

    def on_message_complete(self):
        self.loop.create_task(
            self.request_handler(self.request, self.write_response))

    # -------------------------------------------- #
    # Responding
    # -------------------------------------------- #

    def write_response(self, response):
        try:
            keep_alive = self.parser.should_keep_alive() \
                            and not self.signal.stopped
            self.transport.write(
                response.output(
                    self.request.version, keep_alive, self.request_timeout))
            if not keep_alive:
                self.transport.close()
            else:
                # Record that we received data
                self._last_request_time = current_time
                self.cleanup()
        except Exception as e:
            self.bail_out(
                "Writing request failed, connection closed {}".format(e))

    def bail_out(self, message):
        log.debug(message)
        self.transport.close()

    def cleanup(self):
        self.parser = None
        self.request = None
        self.url = None
        self.headers = None
        self._total_request_size = 0

    def close_if_idle(self):
        """
        Close the connection if a request is not being sent or received
        :return: boolean - True if closed, false if staying open
        """
        if not self.parser:
            self.transport.close()
            return True
        return False

def update_current_time(loop):
    """
    Caches the current time, since it is needed
    at the end of every keep-alive request to update the request timeout time
    :param loop:
    :return:
    """
    global current_time
    current_time = time()
    loop.call_later(1, partial(update_current_time, loop))

def trigger_events(events, loop):
    """
    :param events: one or more sync or async functions to execute
    :param loop: event loop
    """
    if events:
        if not isinstance(events, list):
            events = [events]
        for event in events:
            result = event(loop)
            if isawaitable(result):
                loop.run_until_complete(result)

def serve(host, port, request_handler, before_start=None, after_start=None,
          before_stop=None, after_stop=None,
          debug=False, request_timeout=60, sock=None,
          request_max_size=None, reuse_port=False, loop=None):
    """
    Starts asynchronous HTTP Server on an individual process.
    :param host: Address to host on
    :param port: Port to host on
    :param request_handler: Sanic request handler with middleware
    :param after_start: Function to be executed after the server starts
    listening. Takes single argument `loop`
    :param before_stop: Function to be executed when a stop signal is
    received before it is respected. Takes single argumenet `loop`
    :param debug: Enables debug output (slows server)
    :param request_timeout: time in seconds
    :param sock: Socket for the server to accept connections from
    :param request_max_size: size in bytes, `None` for no limit
    :param reuse_port: `True` for multiple workers
    :param loop: asyncio compatible event loop
    :return: Nothing
    """
    loop = loop or async_loop.new_event_loop()
    asyncio.set_event_loop(loop)

    if debug:
        loop.set_debug(debug)

    trigger_events(before_start, loop)

    connections = {}
    signal = Signal()
    server_coroutine = loop.create_server(lambda: HttpProtocol(
        loop=loop,
        connections=connections,
        signal=signal,
        request_handler=request_handler,
        request_timeout=request_timeout,
        request_max_size=request_max_size,
    ), host, port, reuse_port=reuse_port, sock=sock)

    # Instead of pulling time at the end of every request,
    # pull it once per minute
    loop.call_soon(partial(update_current_time, loop))

    try:
        http_server = loop.run_until_complete(server_coroutine)
    except Exception:
        log.exception("Unable to start server")
        return

    trigger_events(after_start, loop)

    # Register signals for graceful termination
    for _signal in (SIGINT, SIGTERM):
        loop.add_signal_handler(_signal, loop.stop)

    try:
        loop.run_forever()
    finally:
        log.info("Stop requested, draining connections...")

        # Run the on_stop function if provided
        trigger_events(before_stop, loop)

        # Wait for event loop to finish and all connections to drain
        http_server.close()
        loop.run_until_complete(http_server.wait_closed())

        # Complete all tasks on the loop
        signal.stopped = True
        for connection in connections.keys():
            connection.close_if_idle()

        while connections:
            loop.run_until_complete(asyncio.sleep(0.1))

        trigger_events(after_stop, loop)

        loop.close()
<|endoftext|>


<commit_before>import datetime

from rest_framework import generics, permissions, status, filters
from rest_framework.generics import get_object_or_404
from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework_bulk import ListBulkCreateAPIView

from silver.models import (MeteredFeatureUnitsLog, Subscription, MeteredFeature,
                           Customer, Plan, Provider)
from silver.api.serializers import (MeteredFeatureUnitsLogSerializer,
                                    CustomerSerializer, SubscriptionSerializer,
                                    SubscriptionDetailSerializer,
                                    PlanSerializer, MeteredFeatureSerializer,
                                    ProviderSerializer)
from silver.utils import get_object_or_None

class PlanList(generics.ListCreateAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = PlanSerializer
    queryset = Plan.objects.all()
    filter_backends = (filters.DjangoFilterBackend,)
    filter_fields = ('enabled', 'private')

class PlanDetail(generics.RetrieveUpdateDestroyAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = PlanSerializer
    model = Plan
    lookup_field = 'pk'

    def patch(self, request, *args, **kwargs):
        plan = get_object_or_404(Plan.objects, pk=self.kwargs.get('pk', None))
        name = request.DATA.get('name', None)
        generate_after = request.DATA.get('generate_after', None)
        due_days = request.DATA.get('due_days', None)
        plan.name = name or plan.name
        plan.generate_after = generate_after or plan.generate_after
        plan.due_days = due_days or plan.due_days
        plan.save()
        return Response(PlanSerializer(plan).data, status=status.HTTP_200_OK)

    def delete(self, request, *args, **kwargs):
        plan = get_object_or_404(Plan.objects, pk=self.kwargs.get('pk', None))
        plan.enabled = False
        plan.save()
        return Response({"enabled": plan.enabled}, status=status.HTTP_200_OK)

class PlanMeteredFeatures(ListBulkCreateAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = MeteredFeatureSerializer
    model = MeteredFeature

    def get_queryset(self):
        plan = get_object_or_None(Plan, pk=self.kwargs['pk'])
        return plan.metered_features.all() if plan else None

class MeteredFeaturesList(ListBulkCreateAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = MeteredFeatureSerializer
    model = MeteredFeature

class MeteredFeaturesDetail(generics.RetrieveUpdateAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = MeteredFeatureSerializer
    model = MeteredFeature

class SubscriptionList(generics.ListCreateAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    model = Subscription
    serializer_class = SubscriptionSerializer

class SubscriptionDetail(generics.RetrieveAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    model = Subscription
    serializer_class = SubscriptionDetailSerializer
    lookup_url_kwarg = 'sub'
    lookup_field = 'pk'

class SubscriptionDetailActivate(APIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)

    def post(self, request, *args, **kwargs):
        sub = get_object_or_404(Subscription.objects,
                                pk=self.kwargs.get('pk', None))
        if sub.state != 'inactive':
            message = 'Cannot activate subscription from %s state.' % sub.state
            return Response({"error": message},
                            status=status.HTTP_400_BAD_REQUEST)
        else:
            if request.POST['_content']:
                start_date = request.DATA.get('start_date', None)
                trial_end = request.DATA.get('trial_end_date', None)
                sub.activate(start_date=start_date, trial_end_date=trial_end)
                sub.save()
            else:
                sub.activate()
                sub.save()
            return Response({"state: %s" % sub.state},
                            status=status.HTTP_200_OK)

class SubscriptionDetailCancel(APIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)

    def post(self, request, *args, **kwargs):
        sub = get_object_or_404(Subscription.objects,
                                pk=self.kwargs.get('sub', None))
        when = request.DATA.get('when', None)
        if sub.state != 'active':
            message = 'Cannot cancel subscription from %s state.' % sub.state
            return Response({"error": message},
                            status=status.HTTP_400_BAD_REQUEST)
        else:
            if when == 'now':
                sub.cancel()
                sub.end()
                sub.save()
                return Response({"state: %s" % sub.state},
                                status=status.HTTP_200_OK)
            elif when == 'end_of_billing_cycle':
                sub.cancel()
                sub.save()
                return Response({"state: %s" % sub.state},
                                status=status.HTTP_200_OK)
            else:
                return Response(status=status.HTTP_400_BAD_REQUEST)

class SubscriptionDetailReactivate(APIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)

    def post(self, request, *args, **kwargs):
        sub = get_object_or_404(Subscription.objects,
                                pk=self.kwargs.get('sub', None))
        if sub.state != 'canceled':
            message = 'Cannot reactivate subscription from %s state.' % sub.state
            return Response({"error": message},
                            status=status.HTTP_400_BAD_REQUEST)
        else:
            sub.activate()
            sub.save()
            return Response({"state: %s" % sub.state},
                            status=status.HTTP_200_OK)

class MeteredFeatureUnitsLogList(generics.ListAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = MeteredFeatureUnitsLogSerializer

    def get_queryset(self):
        return MeteredFeatureUnitsLog.objects.filter(
            metered_feature=self.kwargs['mf'],
            subscription=self.kwargs['sub'],
        )

    def patch(self, request, *args, **kwargs):
        metered_feature_pk = self.kwargs['mf']
        subscription_pk = self.kwargs['sub']
        date = request.DATA.get('date', None)
        consumed_units = request.DATA.get('count', None)
        update_type = request.DATA.get('update_type', None)

        if subscription_pk and metered_feature_pk:
            subscription = get_object_or_None(pk=subscription_pk)
            metered_feature = get_object_or_None(pk=metered_feature_pk)

            if subscription and metered_feature:
                if subscription.state != 'active':
                    return Response({"detail": "Subscription is not active"},
                                    status=status.HTTP_403_FORBIDDEN)
                if date and consumed_units is not None and update_type:
                    try:
                        date = datetime.datetime.strptime(date,
                                                          '%Y-%m-%d').date()
                        if subscription.current_start_date <= date <= \
                           subscription.current_end_date:
                            try:
                                log = MeteredFeatureUnitsLog.objects.get(
                                    start_date__lte=date,
                                    end_date__gte=date,
                                    metered_feature=metered_feature_pk,
                                    subscription=subscription_pk
                                )
                                if update_type == 'absolute':
                                    log.consumed_units = consumed_units
                                elif update_type == 'relative':
                                    log.consumed_units += consumed_units
                                log.save()
                            except MeteredFeatureUnitsLog.DoesNotExist:
                                log = MeteredFeatureUnitsLog.objects.create(
                                    metered_feature=metered_feature,
                                    subscription=subscription,
                                    start_date=subscription.current_start_date,
                                    end_date=subscription.current_end_date,
                                    consumed_units=consumed_units
                                )
                            finally:
                                return Response({"count": log.consumed_units},
                                            status=status.HTTP_200_OK)
                        else:
                            return Response({"detail": "Date is out of bounds"},
                                            status=status.HTTP_400_BAD_REQUEST)
                    except TypeError:
                        return Response({"detail": "Invalid date format"},
                                        status=status.HTTP_400_BAD_REQUEST)
                else:
                    return Response({"detail": "Not enough information provided"},
                                    status=status.HTTP_400_BAD_REQUEST)
            else:
                return Response({"detail": "Not found"},
                                status=status.HTTP_404_NOT_FOUND)
        return Response({"detail": "Wrong address"},
                        status=status.HTTP_500_INTERNAL_SERVER_ERROR)

class CustomerList(generics.ListCreateAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = CustomerSerializer
    model = Customer

class CustomerDetail(generics.RetrieveUpdateAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = CustomerSerializer
    model = Customer
    lookup_field = 'pk'

class ProviderList(ListBulkCreateAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = ProviderSerializer
    model = Provider

class ProviderDetail(generics.RetrieveUpdateAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = ProviderSerializer
    model = Provider
<commit_msg>Remove the lookup_field which is redundant (by default the lookup_field is pk).<commit_after>import datetime

from rest_framework import generics, permissions, status, filters
from rest_framework.generics import get_object_or_404
from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework_bulk import ListBulkCreateAPIView

from silver.models import (MeteredFeatureUnitsLog, Subscription, MeteredFeature,
                           Customer, Plan, Provider)
from silver.api.serializers import (MeteredFeatureUnitsLogSerializer,
                                    CustomerSerializer, SubscriptionSerializer,
                                    SubscriptionDetailSerializer,
                                    PlanSerializer, MeteredFeatureSerializer,
                                    ProviderSerializer)
from silver.utils import get_object_or_None

class PlanList(generics.ListCreateAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = PlanSerializer
    queryset = Plan.objects.all()
    filter_backends = (filters.DjangoFilterBackend,)
    filter_fields = ('enabled', 'private')

class PlanDetail(generics.RetrieveUpdateDestroyAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = PlanSerializer
    model = Plan
    lookup_field = 'pk'

    def patch(self, request, *args, **kwargs):
        plan = get_object_or_404(Plan.objects, pk=self.kwargs.get('pk', None))
        name = request.DATA.get('name', None)
        generate_after = request.DATA.get('generate_after', None)
        due_days = request.DATA.get('due_days', None)
        plan.name = name or plan.name
        plan.generate_after = generate_after or plan.generate_after
        plan.due_days = due_days or plan.due_days
        plan.save()
        return Response(PlanSerializer(plan).data, status=status.HTTP_200_OK)

    def delete(self, request, *args, **kwargs):
        plan = get_object_or_404(Plan.objects, pk=self.kwargs.get('pk', None))
        plan.enabled = False
        plan.save()
        return Response({"enabled": plan.enabled}, status=status.HTTP_200_OK)

class PlanMeteredFeatures(ListBulkCreateAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = MeteredFeatureSerializer
    model = MeteredFeature

    def get_queryset(self):
        plan = get_object_or_None(Plan, pk=self.kwargs['pk'])
        return plan.metered_features.all() if plan else None

class MeteredFeaturesList(ListBulkCreateAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = MeteredFeatureSerializer
    model = MeteredFeature

class MeteredFeaturesDetail(generics.RetrieveUpdateAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = MeteredFeatureSerializer
    model = MeteredFeature

class SubscriptionList(generics.ListCreateAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    model = Subscription
    serializer_class = SubscriptionSerializer

class SubscriptionDetail(generics.RetrieveAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    model = Subscription
    serializer_class = SubscriptionDetailSerializer
    lookup_url_kwarg = 'sub'
    lookup_field = 'pk'

class SubscriptionDetailActivate(APIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)

    def post(self, request, *args, **kwargs):
        sub = get_object_or_404(Subscription.objects,
                                pk=self.kwargs.get('pk', None))
        if sub.state != 'inactive':
            message = 'Cannot activate subscription from %s state.' % sub.state
            return Response({"error": message},
                            status=status.HTTP_400_BAD_REQUEST)
        else:
            if request.POST['_content']:
                start_date = request.DATA.get('start_date', None)
                trial_end = request.DATA.get('trial_end_date', None)
                sub.activate(start_date=start_date, trial_end_date=trial_end)
                sub.save()
            else:
                sub.activate()
                sub.save()
            return Response({"state: %s" % sub.state},
                            status=status.HTTP_200_OK)

class SubscriptionDetailCancel(APIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)

    def post(self, request, *args, **kwargs):
        sub = get_object_or_404(Subscription.objects,
                                pk=self.kwargs.get('sub', None))
        when = request.DATA.get('when', None)
        if sub.state != 'active':
            message = 'Cannot cancel subscription from %s state.' % sub.state
            return Response({"error": message},
                            status=status.HTTP_400_BAD_REQUEST)
        else:
            if when == 'now':
                sub.cancel()
                sub.end()
                sub.save()
                return Response({"state: %s" % sub.state},
                                status=status.HTTP_200_OK)
            elif when == 'end_of_billing_cycle':
                sub.cancel()
                sub.save()
                return Response({"state: %s" % sub.state},
                                status=status.HTTP_200_OK)
            else:
                return Response(status=status.HTTP_400_BAD_REQUEST)

class SubscriptionDetailReactivate(APIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)

    def post(self, request, *args, **kwargs):
        sub = get_object_or_404(Subscription.objects,
                                pk=self.kwargs.get('sub', None))
        if sub.state != 'canceled':
            message = 'Cannot reactivate subscription from %s state.' % sub.state
            return Response({"error": message},
                            status=status.HTTP_400_BAD_REQUEST)
        else:
            sub.activate()
            sub.save()
            return Response({"state: %s" % sub.state},
                            status=status.HTTP_200_OK)

class MeteredFeatureUnitsLogList(generics.ListAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = MeteredFeatureUnitsLogSerializer

    def get_queryset(self):
        return MeteredFeatureUnitsLog.objects.filter(
            metered_feature=self.kwargs['mf'],
            subscription=self.kwargs['sub'],
        )

    def patch(self, request, *args, **kwargs):
        metered_feature_pk = self.kwargs['mf']
        subscription_pk = self.kwargs['sub']
        date = request.DATA.get('date', None)
        consumed_units = request.DATA.get('count', None)
        update_type = request.DATA.get('update_type', None)

        if subscription_pk and metered_feature_pk:
            subscription = get_object_or_None(pk=subscription_pk)
            metered_feature = get_object_or_None(pk=metered_feature_pk)

            if subscription and metered_feature:
                if subscription.state != 'active':
                    return Response({"detail": "Subscription is not active"},
                                    status=status.HTTP_403_FORBIDDEN)
                if date and consumed_units is not None and update_type:
                    try:
                        date = datetime.datetime.strptime(date,
                                                          '%Y-%m-%d').date()
                        if subscription.current_start_date <= date <= \
                           subscription.current_end_date:
                            try:
                                log = MeteredFeatureUnitsLog.objects.get(
                                    start_date__lte=date,
                                    end_date__gte=date,
                                    metered_feature=metered_feature_pk,
                                    subscription=subscription_pk
                                )
                                if update_type == 'absolute':
                                    log.consumed_units = consumed_units
                                elif update_type == 'relative':
                                    log.consumed_units += consumed_units
                                log.save()
                            except MeteredFeatureUnitsLog.DoesNotExist:
                                log = MeteredFeatureUnitsLog.objects.create(
                                    metered_feature=metered_feature,
                                    subscription=subscription,
                                    start_date=subscription.current_start_date,
                                    end_date=subscription.current_end_date,
                                    consumed_units=consumed_units
                                )
                            finally:
                                return Response({"count": log.consumed_units},
                                            status=status.HTTP_200_OK)
                        else:
                            return Response({"detail": "Date is out of bounds"},
                                            status=status.HTTP_400_BAD_REQUEST)
                    except TypeError:
                        return Response({"detail": "Invalid date format"},
                                        status=status.HTTP_400_BAD_REQUEST)
                else:
                    return Response({"detail": "Not enough information provided"},
                                    status=status.HTTP_400_BAD_REQUEST)
            else:
                return Response({"detail": "Not found"},
                                status=status.HTTP_404_NOT_FOUND)
        return Response({"detail": "Wrong address"},
                        status=status.HTTP_500_INTERNAL_SERVER_ERROR)

class CustomerList(generics.ListCreateAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = CustomerSerializer
    model = Customer

class CustomerDetail(generics.RetrieveUpdateAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = CustomerSerializer
    model = Customer

class ProviderList(ListBulkCreateAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = ProviderSerializer
    model = Provider

class ProviderDetail(generics.RetrieveUpdateAPIView):
    permission_classes = (permissions.IsAuthenticated, permissions.IsAdminUser,)
    serializer_class = ProviderSerializer
    model = Provider
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
from contextlib import contextmanager

from mock import patch, Mock
from unittest import TestCase
import stripe

from . import StripeProvider
from .. import RedirectNeeded

SECRET_KEY = '1234abcd'
PUBLIC_KEY = 'abcd1234'

class Payment(Mock):

    id = 1
    description = 'payment'
    currency = 'USD'
    delivery = 10
    status = 'waiting'
    message = None
    tax = 10
    total = 100
    captured_amount = 0
    transaction_id = None

    def change_status(self, status, message=''):
        self.status = status
        self.message = message

    def change_fraud_status(self, status, message='', commit=True):
        self.fraud_status = status
        self.fraud_message = message

    def get_failure_url(self):
        return 'http://cancel.com'

    def get_process_url(self):
        return 'http://example.com'

    def get_purchased_items(self):
        return []

    def get_success_url(self):
        return 'http://success.com'

@contextmanager
def mock_stripe_Charge_create(error_msg=None):
    json_body = {
        'error': {
            'charge': 'charge_id'
        }
    }
    with patch('stripe.Charge.create') as mocked_charge_create:
        if error_msg:
            mocked_charge_create.side_effect = stripe.CardError(
                error_msg, param=None, code=None, json_body=json_body)
        else:
            mocked_charge_create.side_effect = lambda **kwargs: {}
        yield mocked_charge_create

@contextmanager
def mock_stripe_Charge_retrieve(fraudulent=False):
    with patch('stripe.Charge.retrieve') as mocked_charge_retrieve:
        fraud_details = {
            'stripe_report': None
        }
        if fraudulent:
            fraud_details['stripe_report'] = 'fraudulent'
        mocked_charge_retrieve.side_effect = lambda charge_id: {
            'id': charge_id,
            'fraud_details': fraud_details
        }
        yield mocked_charge_retrieve

class TestStripeProvider(TestCase):

    def test_form_contains_stripe_script(self):
        payment = Payment()
        store_name = 'Test store'
        provider = StripeProvider(
            name=store_name,
            secret_key=SECRET_KEY, public_key=PUBLIC_KEY)
        form = provider.get_form(payment)
        self.assertTrue(
            '<script class="stripe-button" data-amount="10000" '
            'data-currency="USD" data-description="payment" data-image="" '
            'data-key="%s" data-name="%s" '
            'src="https://checkout.stripe.com/checkout.js"></script>' % (
                PUBLIC_KEY, store_name)
            in str(form))

    def test_provider_raises_redirect_needed_when_token_does_not_exist(self):
        payment = Payment()
        provider = StripeProvider(
            name='Example.com store',
            secret_key=SECRET_KEY, public_key=PUBLIC_KEY)
        data = {}
        with self.assertRaises(RedirectNeeded) as exc:
            provider.get_form(payment, data)
            self.assertEqual(exc.args[0], payment.get_failure_url())
        self.assertEqual(payment.status, 'rejected')

    def test_provider_raises_redirect_needed_on_success(self):
        payment = Payment()
        provider = StripeProvider(
            name='Example.com store',
            secret_key=SECRET_KEY, public_key=PUBLIC_KEY)
        data = {'stripeToken': 'abcd'}
        with patch('stripe.Charge.create'):
            with self.assertRaises(RedirectNeeded) as exc:
                provider.get_form(payment, data)
                self.assertEqual(exc.args[0], payment.get_success_url())
        self.assertEqual(payment.status, 'confirmed')
        self.assertEqual(payment.captured_amount, payment.total)

    def test_provider_shows_validation_error_message(self):
        error_msg = 'Error message'

        payment = Payment()
        provider = StripeProvider(
            name='Example.com store',
            secret_key=SECRET_KEY, public_key=PUBLIC_KEY)
        data = {'stripeToken': 'abcd'}
        with mock_stripe_Charge_create(error_msg=error_msg):
            with mock_stripe_Charge_retrieve():
                form = provider.get_form(payment, data=data)
                self.assertEqual(form.errors['__all__'][0], error_msg)
        self.assertEqual(payment.status, 'error')
        self.assertEqual(payment.message, error_msg)
        self.assertEqual(payment.captured_amount, 0)

    def test_provider_set_fraudulent_status(self):
        error_msg = 'Error message'
        payment = Payment()
        provider = StripeProvider(
            name='Example.com store',
            secret_key=SECRET_KEY, public_key=PUBLIC_KEY)
        data = {'stripeToken': 'abcd'}
        with mock_stripe_Charge_create(error_msg=error_msg):
            with mock_stripe_Charge_retrieve(fraudulent=True):
                provider.get_form(payment, data=data)
        self.assertEqual(payment.status, 'error')
        self.assertEqual(payment.fraud_status, 'reject')
        self.assertEqual(payment.captured_amount, 0)

    def test_provider_detect_already_processed_payment(self):
        payment = Payment()
        payment.transaction_id = 'existing_transaction_id'
        provider = StripeProvider(
            name='Example.com store',
            secret_key=SECRET_KEY, public_key=PUBLIC_KEY)
        data = {'stripeToken': 'abcd'}
        with mock_stripe_Charge_create():
            with mock_stripe_Charge_retrieve():
                form = provider.get_form(payment, data=data)
                msg = 'This payment has already been processed.'
                self.assertEqual(form.errors['__all__'][0], msg)
<commit_msg>Fix stripe tests<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
from contextlib import contextmanager

from mock import patch, Mock
from unittest import TestCase
import stripe

from . import StripeProvider
from .. import RedirectNeeded

SECRET_KEY = '1234abcd'
PUBLIC_KEY = 'abcd1234'

class Payment(Mock):

    id = 1
    description = 'payment'
    currency = 'USD'
    delivery = 10
    status = 'waiting'
    message = None
    tax = 10
    total = 100
    captured_amount = 0
    transaction_id = None

    def change_status(self, status, message=''):
        self.status = status
        self.message = message

    def change_fraud_status(self, status, message='', commit=True):
        self.fraud_status = status
        self.fraud_message = message

    def capture(self, amount=None):
        amount = amount or self.total
        self.captured_amount = amount
        self.change_status('confirmed')

    def get_failure_url(self):
        return 'http://cancel.com'

    def get_process_url(self):
        return 'http://example.com'

    def get_purchased_items(self):
        return []

    def get_success_url(self):
        return 'http://success.com'

@contextmanager
def mock_stripe_Charge_create(error_msg=None):
    json_body = {
        'error': {
            'charge': 'charge_id'
        }
    }
    with patch('stripe.Charge.create') as mocked_charge_create:
        if error_msg:
            mocked_charge_create.side_effect = stripe.CardError(
                error_msg, param=None, code=None, json_body=json_body)
        else:
            mocked_charge_create.side_effect = lambda **kwargs: {}
        yield mocked_charge_create

@contextmanager
def mock_stripe_Charge_retrieve(fraudulent=False):
    with patch('stripe.Charge.retrieve') as mocked_charge_retrieve:
        fraud_details = {
            'stripe_report': None
        }
        if fraudulent:
            fraud_details['stripe_report'] = 'fraudulent'
        mocked_charge_retrieve.side_effect = lambda charge_id: {
            'id': charge_id,
            'fraud_details': fraud_details
        }
        yield mocked_charge_retrieve

class TestStripeProvider(TestCase):

    def test_form_contains_stripe_script(self):
        payment = Payment()
        store_name = 'Test store'
        provider = StripeProvider(
            name=store_name,
            secret_key=SECRET_KEY, public_key=PUBLIC_KEY)
        form = provider.get_form(payment)
        self.assertTrue(
            '<script class="stripe-button" data-amount="10000" '
            'data-currency="USD" data-description="payment" data-image="" '
            'data-key="%s" data-name="%s" '
            'src="https://checkout.stripe.com/checkout.js"></script>' % (
                PUBLIC_KEY, store_name)
            in str(form))

    def test_provider_raises_redirect_needed_when_token_does_not_exist(self):
        payment = Payment()
        provider = StripeProvider(
            name='Example.com store',
            secret_key=SECRET_KEY, public_key=PUBLIC_KEY)
        data = {}
        with self.assertRaises(RedirectNeeded) as exc:
            provider.get_form(payment, data)
            self.assertEqual(exc.args[0], payment.get_failure_url())
        self.assertEqual(payment.status, 'rejected')

    def test_provider_raises_redirect_needed_on_success(self):
        payment = Payment()
        provider = StripeProvider(
            name='Example.com store',
            secret_key=SECRET_KEY, public_key=PUBLIC_KEY)
        data = {'stripeToken': 'abcd'}
        with patch('stripe.util.json.dumps'):
            with patch('stripe.Charge.create'):
                with self.assertRaises(RedirectNeeded) as exc:
                    provider.get_form(payment, data)
                    self.assertEqual(exc.args[0], payment.get_success_url())
        self.assertEqual(payment.status, 'confirmed')
        self.assertEqual(payment.captured_amount, payment.total)

    def test_provider_shows_validation_error_message(self):
        error_msg = 'Error message'

        payment = Payment()
        provider = StripeProvider(
            name='Example.com store',
            secret_key=SECRET_KEY, public_key=PUBLIC_KEY)
        data = {'stripeToken': 'abcd'}
        with mock_stripe_Charge_create(error_msg=error_msg):
            with mock_stripe_Charge_retrieve():
                form = provider.get_form(payment, data=data)
                self.assertEqual(form.errors['__all__'][0], error_msg)
        self.assertEqual(payment.status, 'error')
        self.assertEqual(payment.message, error_msg)
        self.assertEqual(payment.captured_amount, 0)

    def test_provider_set_fraudulent_status(self):
        error_msg = 'Error message'
        payment = Payment()
        provider = StripeProvider(
            name='Example.com store',
            secret_key=SECRET_KEY, public_key=PUBLIC_KEY)
        data = {'stripeToken': 'abcd'}
        with mock_stripe_Charge_create(error_msg=error_msg):
            with mock_stripe_Charge_retrieve(fraudulent=True):
                provider.get_form(payment, data=data)
        self.assertEqual(payment.status, 'error')
        self.assertEqual(payment.fraud_status, 'reject')
        self.assertEqual(payment.captured_amount, 0)

    def test_provider_detect_already_processed_payment(self):
        payment = Payment()
        payment.transaction_id = 'existing_transaction_id'
        provider = StripeProvider(
            name='Example.com store',
            secret_key=SECRET_KEY, public_key=PUBLIC_KEY)
        data = {'stripeToken': 'abcd'}
        with mock_stripe_Charge_create():
            with mock_stripe_Charge_retrieve():
                form = provider.get_form(payment, data=data)
                msg = 'This payment has already been processed.'
                self.assertEqual(form.errors['__all__'][0], msg)
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

import os

from pyro.distributions.binomial import Binomial
from pyro.distributions.cauchy import Cauchy
from pyro.distributions.delta import Delta
from pyro.distributions.distribution import Distribution  # noqa: F401
from pyro.distributions.half_cauchy import HalfCauchy
from pyro.distributions.log_normal import LogNormal
from pyro.distributions.multinomial import Multinomial
from pyro.distributions.poisson import Poisson
from pyro.distributions.random_primitive import RandomPrimitive
from pyro.distributions.uniform import Uniform

# TODO Decide based on torch.__version__ once torch.distributions matures.
USE_TORCH_DISTRIBUTIONS = int(os.environ.get('PYRO_USE_TORCH_DISTRIBUTIONS', 0))

# distribution classes with working torch versions
if USE_TORCH_DISTRIBUTIONS:
    from pyro.distributions.torch.bernoulli import Bernoulli
    from pyro.distributions.torch.beta import Beta
    from pyro.distributions.torch.categorical import Categorical
    from pyro.distributions.torch.dirichlet import Dirichlet
    from pyro.distributions.torch.exponential import Exponential
    from pyro.distributions.torch.gamma import Gamma
    from pyro.distributions.torch.normal import Normal
    from pyro.distributions.torch.one_hot_categorical import OneHotCategorical
else:
    from pyro.distributions.bernoulli import Bernoulli
    from pyro.distributions.beta import Beta
    from pyro.distributions.categorical import Categorical
    from pyro.distributions.dirichlet import Dirichlet
    from pyro.distributions.exponential import Exponential
    from pyro.distributions.gamma import Gamma
    from pyro.distributions.normal import Normal
    from pyro.distributions.one_hot_categorical import OneHotCategorical

# function aliases
bernoulli = RandomPrimitive(Bernoulli)
beta = RandomPrimitive(Beta)
binomial = RandomPrimitive(Binomial)
categorical = RandomPrimitive(Categorical)
cauchy = RandomPrimitive(Cauchy)
delta = RandomPrimitive(Delta)
dirichlet = RandomPrimitive(Dirichlet)
exponential = RandomPrimitive(Exponential)
gamma = RandomPrimitive(Gamma)
halfcauchy = RandomPrimitive(HalfCauchy)
lognormal = RandomPrimitive(LogNormal)
multinomial = RandomPrimitive(Multinomial)
normal = RandomPrimitive(Normal)
one_hot_categorical = RandomPrimitive(OneHotCategorical)
poisson = RandomPrimitive(Poisson)
uniform = RandomPrimitive(Uniform)
<commit_msg>Add contributors comment to distributions/__init__.py (#654)<commit_after>from __future__ import absolute_import, division, print_function

import os

# Notice to Contributors. (@fritzo 2017-12-26)
#
# The Pyro team is moving pyro.distributions implementations upstream to
# torch.distributions, aiming for the PyTorch 0.4 release and Pyro 0.2 release
# in late Jan or Feb 2018.
# Tasks: https://github.com/probtorch/pytorch/projects/1
# Design Doc: https://goo.gl/9ccYsq
#
# To contribute new distributions you can either:
# 1. (Preferred) Implement a new distributions in torch.distributions and then
#    create a wrapper in pyro.distributions.torch.
# 2. Implement a new distribution in pyro.distribution and let Pyro devs move
#    this implementation upstream to torch.distributions.

# TODO move these implementations upstream to torch.distributions
from pyro.distributions.binomial import Binomial
from pyro.distributions.cauchy import Cauchy
from pyro.distributions.delta import Delta
from pyro.distributions.distribution import Distribution  # noqa: F401
from pyro.distributions.half_cauchy import HalfCauchy
from pyro.distributions.log_normal import LogNormal
from pyro.distributions.multinomial import Multinomial
from pyro.distributions.poisson import Poisson
from pyro.distributions.random_primitive import RandomPrimitive
from pyro.distributions.uniform import Uniform

# distribution classes with working torch versions in torch.distributions
USE_TORCH_DISTRIBUTIONS = int(os.environ.get('PYRO_USE_TORCH_DISTRIBUTIONS', 0))
if USE_TORCH_DISTRIBUTIONS:
    from pyro.distributions.torch.bernoulli import Bernoulli
    from pyro.distributions.torch.beta import Beta
    from pyro.distributions.torch.categorical import Categorical
    from pyro.distributions.torch.dirichlet import Dirichlet
    from pyro.distributions.torch.exponential import Exponential
    from pyro.distributions.torch.gamma import Gamma
    from pyro.distributions.torch.normal import Normal
    from pyro.distributions.torch.one_hot_categorical import OneHotCategorical
else:
    from pyro.distributions.bernoulli import Bernoulli
    from pyro.distributions.beta import Beta
    from pyro.distributions.categorical import Categorical
    from pyro.distributions.dirichlet import Dirichlet
    from pyro.distributions.exponential import Exponential
    from pyro.distributions.gamma import Gamma
    from pyro.distributions.normal import Normal
    from pyro.distributions.one_hot_categorical import OneHotCategorical

# function aliases
bernoulli = RandomPrimitive(Bernoulli)
beta = RandomPrimitive(Beta)
binomial = RandomPrimitive(Binomial)
categorical = RandomPrimitive(Categorical)
cauchy = RandomPrimitive(Cauchy)
delta = RandomPrimitive(Delta)
dirichlet = RandomPrimitive(Dirichlet)
exponential = RandomPrimitive(Exponential)
gamma = RandomPrimitive(Gamma)
halfcauchy = RandomPrimitive(HalfCauchy)
lognormal = RandomPrimitive(LogNormal)
multinomial = RandomPrimitive(Multinomial)
normal = RandomPrimitive(Normal)
one_hot_categorical = RandomPrimitive(OneHotCategorical)
poisson = RandomPrimitive(Poisson)
uniform = RandomPrimitive(Uniform)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Supports the Ion Velocity Meter (IVM) onboard the Communication
and Navigation Outage Forecasting System (C/NOFS) satellite, part
of the Coupled Ion Netural Dynamics Investigation (CINDI). Downloads
data from the NASA Coordinated Data Analysis Web (CDAWeb) in CDF
format.

The IVM is composed of the Retarding Potential Analyzer (RPA) and
Drift Meter (DM). The RPA measures the energy of plasma along the
direction of satellite motion. By fitting these measurements
to a theoretical description of plasma the number density, plasma
composition, plasma temperature, and plasma motion may be determined.
The DM directly measures the arrival angle of plasma. Using the reported
motion of the satellite the angle is converted into ion motion along
two orthogonal directions, perpendicular to the satellite track.

A brief discussion of the C/NOFS mission and instruments can be found at
de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast
scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591,
doi:10.1016/j.jastp.2004.07.030.

Parameters
----------
platform : string
    'cnofs'
name : string
    'ivm'
tag : string
    None supported
sat_id : string
    None supported

Warnings
--------
- The sampling rate of the instrument changes on July 29th, 2010.
  The rate is attached to the instrument object as .sample_rate.

- The cleaning parameters for the instrument are still under development.

"""
from __future__ import print_function
from __future__ import absolute_import

import functools

import numpy as np

import pysat

from .methods import nasa_cdaweb as cdw

platform = 'cnofs'
name = 'ivm'
tags = {'': ''}
sat_ids = {'': ['']}
test_dates = {'': {'': pysat.datetime(2009, 1, 1)}}

# support list files routine
# use the default CDAWeb method
fname = 'cnofs_cindi_ivm_500ms_{year:4d}{month:02d}{day:02d}_v01.cdf'
supported_tags = {'': {'': fname}}
list_files = functools.partial(cdw.list_files,
                               supported_tags=supported_tags)

# support load routine
# use the default CDAWeb method
load = cdw.load

# support download routine
# use the default CDAWeb method
basic_tag = {'dir': '/pub/data/cnofs/cindi/ivm_500ms_cdf',
             'remote_fname': '{year:4d}/' + fname,
             'local_fname': fname}
supported_tags = {'': {'': basic_tag}}
download = functools.partial(cdw.download, supported_tags)
# support listing files currently on CDAWeb
list_remote_files = functools.partial(cdw.list_remote_files,
                                      supported_tags=supported_tags)

def default(ivm):
    ivm.sample_rate = 1.0 if ivm.date >= pysat.datetime(2010, 7, 29) else 2.0

def clean(inst):
    """Routine to return C/NOFS IVM data cleaned to the specified level

    Parameters
    -----------
    inst : (pysat.Instrument)
        Instrument class object, whose attribute clean_level is used to return
        the desired level of data selectivity.

    Returns
    --------
    Void : (NoneType)
        data in inst is modified in-place.

    Notes
    --------
    Supports 'clean', 'dusty', 'dirty'

    """

    # make sure all -999999 values are NaN
    inst.data.replace(-999999., np.nan, inplace=True)

    # Set maximum flags
    if inst.clean_level == 'clean':
        max_rpa_flag = 1
        max_dm_flag = 0
    elif inst.clean_level == 'dusty':
        max_rpa_flag = 3
        max_dm_flag = 3
    else:
        max_rpa_flag = 4
        max_dm_flag = 6

    # First pass, remove bad densities
    idx, = np.where(inst.data.RPAflag <= max_rpa_flag)
    inst.data = inst[idx, :]

    # Second pass, find bad drifts, replace with NaNs
    idx, = np.where(inst.data.driftMeterflag > max_dm_flag)

    # Also exclude very large drifts and drifts where 100% O+
    if (inst.clean_level == 'clean') | (inst.clean_level == 'dusty'):
        try:
            # unrealistic velocities
            idx2, = np.where(np.abs(inst.data.ionVelmeridional) >= 10000.)
            # shallow fit region for vx
            idx3, = np.where(inst.data.ion1fraction >= 1.0)
            if len(idx2)>0:
                if len(idx)>0:
                    idx = np.unique(np.append(idx, idx2))
                else:
                    idx = idx2
            if len(idx3)>0:
                if len(idx)>0:
                    idx = np.unique(np.append(idx, idx3))
                else:
                    idx = idx3
        except AttributeError:
            pass

    if len(idx)>0:
        drift_labels = ['ionVelmeridional', 'ionVelparallel', 'ionVelzonal',
                        'ionVelocityX', 'ionVelocityY', 'ionVelocityZ']
        for label in drift_labels:
            inst[label][idx] = np.NaN

    # Check for bad temperature fits (O+ < 15%), replace with NaNs
    # Criteria from Hairston et al, 2015
    idx, = np.where(inst.data.ion1fraction < 0.15)
    inst['ionTemperature'][idx] = np.NaN

    # basic quality check on drifts and don't let UTS go above 86400.
    idx, = np.where(inst.data.time <= 86400.)
    inst.data = inst[idx, :]

    # make sure MLT is between 0 and 24
    idx, = np.where((inst.data.mlt >= 0) & (inst.data.mlt <= 24.))
    inst.data = inst[idx, :]
    return
<commit_msg>STY: pythonic check for empty indices<commit_after># -*- coding: utf-8 -*-
"""Supports the Ion Velocity Meter (IVM) onboard the Communication
and Navigation Outage Forecasting System (C/NOFS) satellite, part
of the Coupled Ion Netural Dynamics Investigation (CINDI). Downloads
data from the NASA Coordinated Data Analysis Web (CDAWeb) in CDF
format.

The IVM is composed of the Retarding Potential Analyzer (RPA) and
Drift Meter (DM). The RPA measures the energy of plasma along the
direction of satellite motion. By fitting these measurements
to a theoretical description of plasma the number density, plasma
composition, plasma temperature, and plasma motion may be determined.
The DM directly measures the arrival angle of plasma. Using the reported
motion of the satellite the angle is converted into ion motion along
two orthogonal directions, perpendicular to the satellite track.

A brief discussion of the C/NOFS mission and instruments can be found at
de La Beaujardière, O., et al. (2004), C/NOFS: A mission to forecast
scintillations, J. Atmos. Sol. Terr. Phys., 66, 1573–1591,
doi:10.1016/j.jastp.2004.07.030.

Parameters
----------
platform : string
    'cnofs'
name : string
    'ivm'
tag : string
    None supported
sat_id : string
    None supported

Warnings
--------
- The sampling rate of the instrument changes on July 29th, 2010.
  The rate is attached to the instrument object as .sample_rate.

- The cleaning parameters for the instrument are still under development.

"""
from __future__ import print_function
from __future__ import absolute_import

import functools

import numpy as np

import pysat

from .methods import nasa_cdaweb as cdw

platform = 'cnofs'
name = 'ivm'
tags = {'': ''}
sat_ids = {'': ['']}
test_dates = {'': {'': pysat.datetime(2009, 1, 1)}}

# support list files routine
# use the default CDAWeb method
fname = 'cnofs_cindi_ivm_500ms_{year:4d}{month:02d}{day:02d}_v01.cdf'
supported_tags = {'': {'': fname}}
list_files = functools.partial(cdw.list_files,
                               supported_tags=supported_tags)

# support load routine
# use the default CDAWeb method
load = cdw.load

# support download routine
# use the default CDAWeb method
basic_tag = {'dir': '/pub/data/cnofs/cindi/ivm_500ms_cdf',
             'remote_fname': '{year:4d}/' + fname,
             'local_fname': fname}
supported_tags = {'': {'': basic_tag}}
download = functools.partial(cdw.download, supported_tags)
# support listing files currently on CDAWeb
list_remote_files = functools.partial(cdw.list_remote_files,
                                      supported_tags=supported_tags)

def default(ivm):
    ivm.sample_rate = 1.0 if ivm.date >= pysat.datetime(2010, 7, 29) else 2.0

def clean(inst):
    """Routine to return C/NOFS IVM data cleaned to the specified level

    Parameters
    -----------
    inst : (pysat.Instrument)
        Instrument class object, whose attribute clean_level is used to return
        the desired level of data selectivity.

    Returns
    --------
    Void : (NoneType)
        data in inst is modified in-place.

    Notes
    --------
    Supports 'clean', 'dusty', 'dirty'

    """

    # make sure all -999999 values are NaN
    inst.data.replace(-999999., np.nan, inplace=True)

    # Set maximum flags
    if inst.clean_level == 'clean':
        max_rpa_flag = 1
        max_dm_flag = 0
    elif inst.clean_level == 'dusty':
        max_rpa_flag = 3
        max_dm_flag = 3
    else:
        max_rpa_flag = 4
        max_dm_flag = 6

    # First pass, remove bad densities
    idx, = np.where(inst.data.RPAflag <= max_rpa_flag)
    inst.data = inst[idx, :]

    # Second pass, find bad drifts, replace with NaNs
    idx, = np.where(inst.data.driftMeterflag > max_dm_flag)

    # Also exclude very large drifts and drifts where 100% O+
    if (inst.clean_level == 'clean') | (inst.clean_level == 'dusty'):
        try:
            # unrealistic velocities
            idx2, = np.where(np.abs(inst.data.ionVelmeridional) >= 10000.)
            # shallow fit region for vx
            idx3, = np.where(inst.data.ion1fraction >= 1.0)
            idx = np.unique(idx.extend(idx2, idx3))
        except AttributeError:
            pass

    if len(idx)>0:
        drift_labels = ['ionVelmeridional', 'ionVelparallel', 'ionVelzonal',
                        'ionVelocityX', 'ionVelocityY', 'ionVelocityZ']
        for label in drift_labels:
            inst[label][idx] = np.NaN

    # Check for bad temperature fits (O+ < 15%), replace with NaNs
    # Criteria from Hairston et al, 2015
    idx, = np.where(inst.data.ion1fraction < 0.15)
    inst['ionTemperature'][idx] = np.NaN

    # basic quality check on drifts and don't let UTS go above 86400.
    idx, = np.where(inst.data.time <= 86400.)
    inst.data = inst[idx, :]

    # make sure MLT is between 0 and 24
    idx, = np.where((inst.data.mlt >= 0) & (inst.data.mlt <= 24.))
    inst.data = inst[idx, :]
    return
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import sys

import pytest

class ExpectedException(Exception):
    pass

@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_call(item):
    outcome = yield
    raises_marker = item.get_marker('raises')
    if raises_marker:
        exception = raises_marker.kwargs.get('exception')
        exception = exception or Exception

        raised_exception = outcome.excinfo[1] if outcome.excinfo else None
        traceback = outcome.excinfo[2] if outcome.excinfo else None
        if isinstance(raised_exception, exception):
            outcome.force_result(None)
        else:
            try:
                raise ExpectedException('Expected exception {}, but it did not raise'.format(exception))
            except(ExpectedException) as ex:
                excinfo = sys.exc_info()
                if traceback:
                    outcome.excinfo = excinfo[:2] + (traceback, )
                else:
                    outcome.excinfo = excinfo
<commit_msg>Remove unused variable<commit_after># -*- coding: utf-8 -*-
import sys

import pytest

class ExpectedException(Exception):
    pass

@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_call(item):
    outcome = yield
    raises_marker = item.get_marker('raises')
    if raises_marker:
        exception = raises_marker.kwargs.get('exception')
        exception = exception or Exception

        raised_exception = outcome.excinfo[1] if outcome.excinfo else None
        traceback = outcome.excinfo[2] if outcome.excinfo else None
        if isinstance(raised_exception, exception):
            outcome.force_result(None)
        else:
            try:
                raise ExpectedException('Expected exception {}, but it did not raise'.format(exception))
            except(ExpectedException):
                excinfo = sys.exc_info()
                if traceback:
                    outcome.excinfo = excinfo[:2] + (traceback, )
                else:
                    outcome.excinfo = excinfo
<|endoftext|>


<commit_before>from kivy.uix.screenmanager import *

import pentai.gui.intro_screen as i_m
from pentai.base.defines import *

import random

class PScreenManager(ScreenManager):
    def __init__(self, *args, **kwargs):
        self.demo = None
        super(PScreenManager, self).__init__(*args, **kwargs)
        self.transition = SlideTransition()
        self.random_transition()
        self.previous = []

    def set_demo(self, d):
        if self.demo:
            self.demo.clean_up()
        self.demo = d

    def show_intro_screen(self):
        self.add_widget(i_m.IntroScreen(name="Intro"))
        self.current = "Intro"

    def push_current(self, screen_name):
        self.previous.append(self.current)
        self.set_current(screen_name)
        if len(self.previous) > 4:
            self.previous[:2] = []

    def set_current(self, screen_name):
        if self.current != screen_name:
            self.random_transition()
            self.current = screen_name

    def return_screen(self):
        self.current = self.previous[-1]
        del self.previous[-1]

    def clear_hist(self):
        self.previous[:] = []

    def random_transition(self):
        trans = self.transition

        dirs = ['right','up','down','left']
        try:
            dirs.remove(self.last_choice)
        except: pass
        dc = random.choice(dirs)
        self.last_choice = dc
        trans.direction = dc

    def in_demo_mode(self):
        return self.demo != None

    def on_touch_down(self, *args, **kwargs):
        if self.in_demo_mode():
            self.demo.interrupt()
        else:
            return super(PScreenManager, self).on_touch_down(*args, **kwargs)

    def on_touch_move(self, *args, **kwargs):
        if self.in_demo_mode():
            pass
        else:
            return super(PScreenManager, self).on_touch_move(*args, **kwargs)

    def on_touch_up(self, *args, **kwargs):
        if self.in_demo_mode():
            pass
        else:
            return super(PScreenManager, self).on_touch_up(*args, **kwargs)

    def leave(self):
        self.current_screen.on_leave()

    def get_all_screens(self):
        from menu_screen import *
        from ai_player_screen import *
        from ai_help_screen import *
        from human_player_screen import *
        from human_help_screen import *
        from setup_screen import *
        from setup_help_screen import *
        from settings_screen import *
        from settings_help_screen import *
        from games_screen import *
        from load_help_screen import *

        screens = [(MenuScreen, "Menu"),
                   (SettingsScreen, "Settings"),
                   (SettingsHelpScreen, "SettingsHelp"),
                   (SetupScreen, "Setup"),
                   (SetupHelpScreen, "GameSetupHelp"),
                   (GamesScreen, "Load"),
                   (LoadHelpScreen, "LoadHelp"),
                   (AIPlayerScreen, "AI"), (AIHelpScreen, "AIHelp"),
                   (HumanPlayerScreen, "Human"), (HumanHelpScreen, "HumanHelp"),
                   ]
        return screens
<commit_msg>Don't use nested import with wildcard<commit_after>from kivy.uix.screenmanager import *

import pentai.gui.intro_screen as i_m
from pentai.base.defines import *

import random

class PScreenManager(ScreenManager):
    def __init__(self, *args, **kwargs):
        self.demo = None
        super(PScreenManager, self).__init__(*args, **kwargs)
        self.transition = SlideTransition()
        self.random_transition()
        self.previous = []

    def set_demo(self, d):
        if self.demo:
            self.demo.clean_up()
        self.demo = d

    def show_intro_screen(self):
        self.add_widget(i_m.IntroScreen(name="Intro"))
        self.current = "Intro"

    def push_current(self, screen_name):
        self.previous.append(self.current)
        self.set_current(screen_name)
        if len(self.previous) > 4:
            self.previous[:2] = []

    def set_current(self, screen_name):
        if self.current != screen_name:
            self.random_transition()
            self.current = screen_name

    def return_screen(self):
        self.current = self.previous[-1]
        del self.previous[-1]

    def clear_hist(self):
        self.previous[:] = []

    def random_transition(self):
        trans = self.transition

        dirs = ['right','up','down','left']
        try:
            dirs.remove(self.last_choice)
        except: pass
        dc = random.choice(dirs)
        self.last_choice = dc
        trans.direction = dc

    def in_demo_mode(self):
        return self.demo != None

    def on_touch_down(self, *args, **kwargs):
        if self.in_demo_mode():
            self.demo.interrupt()
        else:
            return super(PScreenManager, self).on_touch_down(*args, **kwargs)

    def on_touch_move(self, *args, **kwargs):
        if self.in_demo_mode():
            pass
        else:
            return super(PScreenManager, self).on_touch_move(*args, **kwargs)

    def on_touch_up(self, *args, **kwargs):
        if self.in_demo_mode():
            pass
        else:
            return super(PScreenManager, self).on_touch_up(*args, **kwargs)

    def leave(self):
        self.current_screen.on_leave()

    def get_all_screens(self):
        import menu_screen as ms_m
        import ai_player_screen as aips_m
        import ai_help_screen as aihs_m
        import human_player_screen as hps_m
        import human_help_screen as hhs_m
        import setup_screen as sts_m
        import setup_help_screen as shs_m
        import settings_screen as ses_m
        import settings_help_screen as sehs_m
        import games_screen as gs_m
        import load_help_screen as lhs_m

        screens = [(ms_m.MenuScreen, "Menu"),
                   (ses_m.SettingsScreen, "Settings"),
                   (sehs_m.SettingsHelpScreen, "SettingsHelp"),
                   (sts_m.SetupScreen, "Setup"),
                   (shs_m.SetupHelpScreen, "GameSetupHelp"),
                   (gs_m.GamesScreen, "Load"),
                   (lhs_m.LoadHelpScreen, "LoadHelp"),
                   (aips_m.AIPlayerScreen, "AI"), (aihs_m.AIHelpScreen, "AIHelp"),
                   (hps_m.HumanPlayerScreen, "Human"), (hhs_m.HumanHelpScreen, "HumanHelp"),
                   ]
        return screens
<|endoftext|>


<commit_before>from optparse import OptionParser

import requests
import urllib2
from jinja2 import Environment, FileSystemLoader
from logger import logger
from requests.exceptions import ConnectionError

from perfrunner.helpers.remote import RemoteHelper
from perfrunner.helpers.rest import SyncGatewayRequestHelper
from perfrunner.settings import ClusterSpec
from perfrunner.settings import TestConfig

class GatewayInstaller(object):

    BUILDS = {
        'http://packages.couchbase.com/builds/mobile/sync_gateway': (
            '0.0.0/{0}/couchbase-sync-gateway_{0}_x86_64-community.rpm',
            '1.0.0/{0}/couchbase-sync-gateway_{0}_x86_64.rpm',
        ),
        'http://packages.couchbase.com.s3.amazonaws.com/builds/mobile/sync_gateway': (
            '0.0.0/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
            '1.0.0/{0}/couchbase-sync-gateway-enterprise_{0}_x86_64.rpm',
            '1.0.1/{0}/couchbase-sync-gateway-enterprise_{0}_x86_64.rpm',
            '1.0.2/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
            '1.0.3/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
            '1.0.4/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
            '1.1.0/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
        ),
        'http://cbfs-ext.hq.couchbase.com/builds': (
            'couchbase-sync-gateway_{}_x86_64-community.rpm',
        ),
        'http://latestbuilds.hq.couchbase.com/couchbase-sync-gateway': (
            '0.0.0/416/couchbase-sync-gateway-enterprise_0.0.0-326_x86_64.rpm',
        ),
        'http://latestbuilds.hq.couchbase.com/couchbase-sync-gateway': (
            '0.0.0/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
            '0.0.1/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
            '1.0.0/{0}/couchbase-sync-gateway-enterprise_{0}_x86_64.rpm',
            '1.0.1/{0}/couchbase-sync-gateway-enterprise_{0}_x86_64.rpm',
            '1.0.2/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
            '1.0.3/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
            '1.0.4/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
            '1.1.0/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
        ),
    }

    def __init__(self, cluster_spec, test_config, options):
        self.remote = RemoteHelper(cluster_spec, test_config, options.verbose)
        self.cluster_spec = cluster_spec
        self.test_config = test_config
        self.version = options.version
        self.request_helper = SyncGatewayRequestHelper()

    def find_package(self):
        for filename, url in self.get_expected_locations():
            try:
                status_code = requests.head(url).status_code
            except ConnectionError:
                pass
            else:
                if status_code == 200:
                    logger.info('Found "{}"'.format(url))
                    return filename, url
        logger.interrupt('Target build not found')

    def get_expected_locations(self):
        for location, patterns in self.BUILDS.items():
            for pattern in patterns:
                url = '{}/{}'.format(location, pattern.format(self.version))
                filename = url.split('/')[-1]
                yield filename, url

    def kill_processes_gateway(self):
        self.remote.kill_processes_gateway()

    def kill_processes_gateload(self):
        self.remote.kill_processes_gateload()

    def uninstall_gateway(self):
        self.remote.uninstall_gateway()
        self.remote.clean_gateway()

    def uninstall_gateload(self):
        self.remote.uninstall_gateload()
        self.remote.clean_gateload()

    def install_gateway(self):
        filename, url = self.find_package()
        self.remote.install_gateway(url, filename)

    def install_gateload(self):
        self.remote.install_gateload()

    def choose_template(self):

        config_url_setting = self.test_config.gateway_settings.config_url
        if len(config_url_setting) > 0:

            target_filename = "downloaded_config.json"
            target_path = "templates/{}".format(target_filename)

            # build url to remote template (http://git.io/b9PK)
            config_url = "http://git.io/{}".format(config_url_setting)

            # download to a file in templates directory
            logger.info("Downloading config: {}".format(config_url))
            contents = urllib2.urlopen(config_url).read()
            logger.info("Writing config to: {}".format(target_path))
            f = open(target_filename, 'w')
            f.write(contents)
            f.close()

            # return name of file
            return target_filename

        if self.test_config.gateway_settings.shadow == 'true':
            return 'gateway_config_shadow_template.json'
        else:
            return 'gateway_config_template.json'

    def generate_sync_gateways_config(self):
        loader = FileSystemLoader('templates')
        env = Environment(loader=loader)

        template_filename = self.choose_template()
        template = env.get_template(template_filename)

        with open('templates/gateway_config.json', 'w') as fh:
            fh.write(template.render(
                conn_in=self.test_config.gateway_settings.conn_in,
                conn_db=self.test_config.gateway_settings.conn_db,
                compression=self.test_config.gateway_settings.compression,
                bucket=self.test_config.buckets[0],
                db_master=self.cluster_spec.yield_masters().next(),
            ))

    def start_sync_gateways(self):
        self.generate_sync_gateways_config()
        self.remote.start_gateway()

    def install(self):
        self.kill_processes_gateway()
        self.uninstall_gateway()
        self.install_gateway()

        self.kill_processes_gateload()
        self.uninstall_gateload()
        self.install_gateload()

        self.start_sync_gateways()
        for idx, gateway_ip in enumerate(self.remote.gateways, start=1):
            self.request_helper.wait_for_gateway_to_start(idx, gateway_ip)
            if self.test_config.gateway_settings.logging_verbose == 'false':
                self.request_helper.turn_off_gateway_logging(gateway_ip)

def main():
    usage = '%prog -c cluster -t test_config -v version'

    parser = OptionParser(usage)

    parser.add_option('-c', dest='cluster_spec_fname',
                      help='path to cluster specification file',
                      metavar='ClusterSpecFilePath')
    parser.add_option('-t', dest='test_config_fname',
                      help='path to test config file',
                      metavar='TestConfigFilePath')
    parser.add_option('-v', dest='version',
                      help='build version', metavar='Version')
    parser.add_option('--verbose', dest='verbose', action='store_true',
                      help='enable verbose logging')

    options, args = parser.parse_args()
    override = args and [arg.split('.') for arg in ' '.join(args).split(',')]

    if not options.cluster_spec_fname or not options.test_config_fname \
            or not options.version:
        parser.error('Missing mandatory parameter')

    cluster_spec = ClusterSpec()
    cluster_spec.parse(options.cluster_spec_fname)
    cluster_spec.verify()

    test_config = TestConfig()
    test_config.parse(options.test_config_fname, override)

    installer = GatewayInstaller(cluster_spec, test_config, options)
    installer.install()

if __name__ == '__main__':
    main()
<commit_msg>It was writing commit to wrong place<commit_after>from optparse import OptionParser

import requests
import urllib2
from jinja2 import Environment, FileSystemLoader
from logger import logger
from requests.exceptions import ConnectionError

from perfrunner.helpers.remote import RemoteHelper
from perfrunner.helpers.rest import SyncGatewayRequestHelper
from perfrunner.settings import ClusterSpec
from perfrunner.settings import TestConfig

class GatewayInstaller(object):

    BUILDS = {
        'http://packages.couchbase.com/builds/mobile/sync_gateway': (
            '0.0.0/{0}/couchbase-sync-gateway_{0}_x86_64-community.rpm',
            '1.0.0/{0}/couchbase-sync-gateway_{0}_x86_64.rpm',
        ),
        'http://packages.couchbase.com.s3.amazonaws.com/builds/mobile/sync_gateway': (
            '0.0.0/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
            '1.0.0/{0}/couchbase-sync-gateway-enterprise_{0}_x86_64.rpm',
            '1.0.1/{0}/couchbase-sync-gateway-enterprise_{0}_x86_64.rpm',
            '1.0.2/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
            '1.0.3/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
            '1.0.4/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
            '1.1.0/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
        ),
        'http://cbfs-ext.hq.couchbase.com/builds': (
            'couchbase-sync-gateway_{}_x86_64-community.rpm',
        ),
        'http://latestbuilds.hq.couchbase.com/couchbase-sync-gateway': (
            '0.0.0/416/couchbase-sync-gateway-enterprise_0.0.0-326_x86_64.rpm',
        ),
        'http://latestbuilds.hq.couchbase.com/couchbase-sync-gateway': (
            '0.0.0/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
            '0.0.1/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
            '1.0.0/{0}/couchbase-sync-gateway-enterprise_{0}_x86_64.rpm',
            '1.0.1/{0}/couchbase-sync-gateway-enterprise_{0}_x86_64.rpm',
            '1.0.2/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
            '1.0.3/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
            '1.0.4/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
            '1.1.0/{0}/couchbase-sync-gateway-community_{0}_x86_64.rpm',
        ),
    }

    def __init__(self, cluster_spec, test_config, options):
        self.remote = RemoteHelper(cluster_spec, test_config, options.verbose)
        self.cluster_spec = cluster_spec
        self.test_config = test_config
        self.version = options.version
        self.request_helper = SyncGatewayRequestHelper()

    def find_package(self):
        for filename, url in self.get_expected_locations():
            try:
                status_code = requests.head(url).status_code
            except ConnectionError:
                pass
            else:
                if status_code == 200:
                    logger.info('Found "{}"'.format(url))
                    return filename, url
        logger.interrupt('Target build not found')

    def get_expected_locations(self):
        for location, patterns in self.BUILDS.items():
            for pattern in patterns:
                url = '{}/{}'.format(location, pattern.format(self.version))
                filename = url.split('/')[-1]
                yield filename, url

    def kill_processes_gateway(self):
        self.remote.kill_processes_gateway()

    def kill_processes_gateload(self):
        self.remote.kill_processes_gateload()

    def uninstall_gateway(self):
        self.remote.uninstall_gateway()
        self.remote.clean_gateway()

    def uninstall_gateload(self):
        self.remote.uninstall_gateload()
        self.remote.clean_gateload()

    def install_gateway(self):
        filename, url = self.find_package()
        self.remote.install_gateway(url, filename)

    def install_gateload(self):
        self.remote.install_gateload()

    def choose_template(self):

        config_url_setting = self.test_config.gateway_settings.config_url
        if len(config_url_setting) > 0:

            target_filename = "downloaded_config.json"
            target_path = "templates/{}".format(target_filename)

            # build url to remote template (http://git.io/b9PK)
            config_url = "http://git.io/{}".format(config_url_setting)

            # download to a file in templates directory
            logger.info("Downloading config: {}".format(config_url))
            contents = urllib2.urlopen(config_url).read()
            logger.info("Writing config to: {}".format(target_path))
            f = open(target_path, 'w')
            f.write(contents)
            f.close()

            # return name of file
            return target_filename

        if self.test_config.gateway_settings.shadow == 'true':
            return 'gateway_config_shadow_template.json'
        else:
            return 'gateway_config_template.json'

    def generate_sync_gateways_config(self):
        loader = FileSystemLoader('templates')
        env = Environment(loader=loader)

        template_filename = self.choose_template()
        template = env.get_template(template_filename)

        with open('templates/gateway_config.json', 'w') as fh:
            fh.write(template.render(
                conn_in=self.test_config.gateway_settings.conn_in,
                conn_db=self.test_config.gateway_settings.conn_db,
                compression=self.test_config.gateway_settings.compression,
                bucket=self.test_config.buckets[0],
                db_master=self.cluster_spec.yield_masters().next(),
            ))

    def start_sync_gateways(self):
        self.generate_sync_gateways_config()
        self.remote.start_gateway()

    def install(self):
        self.kill_processes_gateway()
        self.uninstall_gateway()
        self.install_gateway()

        self.kill_processes_gateload()
        self.uninstall_gateload()
        self.install_gateload()

        self.start_sync_gateways()
        for idx, gateway_ip in enumerate(self.remote.gateways, start=1):
            self.request_helper.wait_for_gateway_to_start(idx, gateway_ip)
            if self.test_config.gateway_settings.logging_verbose == 'false':
                self.request_helper.turn_off_gateway_logging(gateway_ip)

def main():
    usage = '%prog -c cluster -t test_config -v version'

    parser = OptionParser(usage)

    parser.add_option('-c', dest='cluster_spec_fname',
                      help='path to cluster specification file',
                      metavar='ClusterSpecFilePath')
    parser.add_option('-t', dest='test_config_fname',
                      help='path to test config file',
                      metavar='TestConfigFilePath')
    parser.add_option('-v', dest='version',
                      help='build version', metavar='Version')
    parser.add_option('--verbose', dest='verbose', action='store_true',
                      help='enable verbose logging')

    options, args = parser.parse_args()
    override = args and [arg.split('.') for arg in ' '.join(args).split(',')]

    if not options.cluster_spec_fname or not options.test_config_fname \
            or not options.version:
        parser.error('Missing mandatory parameter')

    cluster_spec = ClusterSpec()
    cluster_spec.parse(options.cluster_spec_fname)
    cluster_spec.verify()

    test_config = TestConfig()
    test_config.parse(options.test_config_fname, override)

    installer = GatewayInstaller(cluster_spec, test_config, options)
    installer.install()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from django.db import connection, models
from django.db.models.expressions import Col
from django.utils.functional import cached_property

class ByteArrayField(models.Field):
    """A simple field type for the Postgres bytea type.

    Represents values as bytearray instances in Python.

    """
    def db_type(self, connection):
        return 'bytea'

    def from_db_value(self, value, expression, connection, context):
        if value is not None:
            return bytearray(value)

class EncryptedField(models.Field):
    """A field wrapper to encrypt any field type.

    @@@ TODO:
    - check lookups via joins
    - handle indexes
    - handle nullability
    - handle default values
    - handle any other field params passed to wrapped field?
    - handle field class-level flags (e.g. empty_strings_allowed)
    - handle custom implementations of various methods (get[_db]_prep*,
      to_python, from_db_value, formfield) on wrapped field

    """
    encrypt_sql_template = "pgp_sym_encrypt(%%s::text, '%(key)s')"
    decrypt_sql_template = "pgp_sym_decrypt(%%s, '%(key)s')::%(dbtype)s"

    def __init__(self, wrapped_field, key, **kwargs):
        self.wrapped_field = wrapped_field
        self.wrapped_type = wrapped_field.db_type(connection)
        self.key = key
        self.encrypt_sql = self.encrypt_sql_template % {'key': key}
        self.decrypt_sql = self.decrypt_sql_template % {
            'key': key, 'dbtype': self.wrapped_type}
        super(EncryptedField, self).__init__(**kwargs)

    def db_type(self, connection):
        return 'bytea'

    def get_placeholder(self, value, compiler, connection):
        return self.encrypt_sql

    @cached_property
    def cached_col(self):
        return DecryptedCol(
            self.model._meta.db_table,
            self,
            self.decrypt_sql,
        )

class DecryptedCol(Col):
    def __init__(self, alias, target, decrypt_sql, output_field=None):
        self.decrypt_sql = decrypt_sql
        super(DecryptedCol, self).__init__(alias, target, output_field)

    def as_sql(self, compiler, connection):
        sql, params = super(DecryptedCol, self).as_sql(compiler, connection)
        return self.decrypt_sql % sql, params
<commit_msg>More TODO updates.<commit_after>from django.db import connection, models
from django.db.models.expressions import Col
from django.utils.functional import cached_property

class ByteArrayField(models.Field):
    """A simple field type for the Postgres bytea type.

    Represents values as bytearray instances in Python.

    """
    def db_type(self, connection):
        return 'bytea'

    def from_db_value(self, value, expression, connection, context):
        if value is not None:
            return bytearray(value)

class EncryptedField(models.Field):
    """A field wrapper to encrypt any field type.

    @@@ TODO:
    - check lookups via joins
    - handle indexes
    - handle nullability
    - handle default values
    - handle any other field params passed to wrapped field?
    - handle field class-level flags (e.g. empty_strings_allowed)
    - handle custom implementations of various methods (get[_db]_prep*,
      to_python, from_db_value, formfield) on wrapped field
    - handle migrations
    - make it easy to write a migration from a non-encrypted field to an
      encrypted field.
    - docs

    """
    encrypt_sql_template = "pgp_sym_encrypt(%%s::text, '%(key)s')"
    decrypt_sql_template = "pgp_sym_decrypt(%%s, '%(key)s')::%(dbtype)s"

    def __init__(self, wrapped_field, key, **kwargs):
        self.wrapped_field = wrapped_field
        self.wrapped_type = wrapped_field.db_type(connection)
        self.key = key
        self.encrypt_sql = self.encrypt_sql_template % {'key': key}
        self.decrypt_sql = self.decrypt_sql_template % {
            'key': key, 'dbtype': self.wrapped_type}
        super(EncryptedField, self).__init__(**kwargs)

    def db_type(self, connection):
        return 'bytea'

    def get_placeholder(self, value, compiler, connection):
        return self.encrypt_sql

    @cached_property
    def cached_col(self):
        return DecryptedCol(
            self.model._meta.db_table,
            self,
            self.decrypt_sql,
        )

class DecryptedCol(Col):
    def __init__(self, alias, target, decrypt_sql, output_field=None):
        self.decrypt_sql = decrypt_sql
        super(DecryptedCol, self).__init__(alias, target, output_field)

    def as_sql(self, compiler, connection):
        sql, params = super(DecryptedCol, self).as_sql(compiler, connection)
        return self.decrypt_sql % sql, params
<|endoftext|>


<commit_before><commit_msg>Comment out debug code<commit_after><|endoftext|>


<commit_before>import colander
from deform.widget import RadioChoiceWidget

from pyramid.view import view_config

from owslib.wps import WebProcessingService
from twitcher.registry import proxy_url

from phoenix.wizard.views import Wizard

SOURCE_TYPES = {
    'wizard_esgf_search': "Earth System Grid (ESGF)",
    #'wizard_swift_login': "Swift Cloud",
    'wizard_threddsservice': "Thredds Catalog Service",
    'wizard_upload': "Local Storage",
    'wizard_solr': "Birdhouse Solr Search"
    }

class SourceSchemaNode(colander.SchemaNode):
    schema_type = colander.String

    def after_bind(self, node, kw):
        values = SOURCE_TYPES.items()
        if not kw['request'].solr_activated:
            values.remove(('wizard_solr', SOURCE_TYPES['wizard_solr']))
        self.widget = RadioChoiceWidget(values=values)

class Schema(colander.MappingSchema):
    source = SourceSchemaNode()

class ChooseSource(Wizard):
    def __init__(self, request):
        super(ChooseSource, self).__init__(
            request, name='wizard_source', title="Choose Data Source")
        wps = WebProcessingService(proxy_url(request, self.wizard_state.get('wizard_wps')['identifier']),
                                        verify=False, skip_caps=True)
        process = wps.describeprocess(self.wizard_state.get('wizard_process')['identifier'])
        for data_input in process.dataInputs:
            if data_input.identifier == self.wizard_state.get('wizard_complex_inputs')['identifier']:
                self.title = "Choose Data Source for %s" % data_input.title
                break
        #self.description = self.wizard_state.get('wizard_complex_inputs')['identifier']

    def breadcrumbs(self):
        breadcrumbs = super(ChooseSource, self).breadcrumbs()
        breadcrumbs.append(dict(route_path=self.request.route_path(self.name), title=self.title))
        return breadcrumbs

    def schema(self):
        return Schema().bind(request=self.request)

    def next_success(self, appstruct):
        self.success(appstruct)
        return self.next( appstruct.get('source') )

    @view_config(route_name='wizard_source', renderer='../templates/wizard/default.pt')
    def view(self):
        return super(ChooseSource, self).view()

<commit_msg>disabled upload wizard, pep8<commit_after>import colander
from deform.widget import RadioChoiceWidget

from pyramid.view import view_config

from owslib.wps import WebProcessingService
from twitcher.registry import proxy_url

from phoenix.wizard.views import Wizard

SOURCE_TYPES = {
    'wizard_esgf_search': "Earth System Grid (ESGF)",
    # 'wizard_swift_login': "Swift Cloud",
    'wizard_threddsservice': "Thredds Catalog Service",
    # 'wizard_upload': "Local Storage",
    'wizard_solr': "Birdhouse Solr Search"
    }

class SourceSchemaNode(colander.SchemaNode):
    schema_type = colander.String
    widget = None

    def after_bind(self, node, kw):
        values = SOURCE_TYPES.items()
        if not kw['request'].solr_activated:
            values.remove(('wizard_solr', SOURCE_TYPES['wizard_solr']))
        self.widget = RadioChoiceWidget(values=values)

class Schema(colander.MappingSchema):
    source = SourceSchemaNode()

class ChooseSource(Wizard):
    def __init__(self, request):
        super(ChooseSource, self).__init__(
            request, name='wizard_source', title="Choose Data Source")
        wps = WebProcessingService(proxy_url(request, self.wizard_state.get('wizard_wps')['identifier']),
                                        verify=False, skip_caps=True)
        process = wps.describeprocess(self.wizard_state.get('wizard_process')['identifier'])
        for data_input in process.dataInputs:
            if data_input.identifier == self.wizard_state.get('wizard_complex_inputs')['identifier']:
                self.title = "Choose Data Source for %s" % data_input.title
                break
        # self.description = self.wizard_state.get('wizard_complex_inputs')['identifier']

    def breadcrumbs(self):
        breadcrumbs = super(ChooseSource, self).breadcrumbs()
        breadcrumbs.append(dict(route_path=self.request.route_path(self.name), title=self.title))
        return breadcrumbs

    def schema(self):
        return Schema().bind(request=self.request)

    def next_success(self, appstruct):
        self.success(appstruct)
        return self.next( appstruct.get('source') )

    @view_config(route_name='wizard_source', renderer='../templates/wizard/default.pt')
    def view(self):
        return super(ChooseSource, self).view()
<|endoftext|>