<commit_before>import sys
import json
from collections import defaultdict

import pyelliptic as ec

from zmq.eventloop import ioloop, zmqstream
import zmq
from multiprocessing import Process
from threading import Thread
ioloop.install()
import traceback

# Default port
DEFAULT_PORT=12345

# Get some command line pars
SEED_URI = False

if len(sys.argv) > 2:
    MY_IP = sys.argv[2]
else:
    MY_IP = "127.0.0.1"
if len(sys.argv) > 3:
    SEED_URI = sys.argv[3] # like tcp://127.0.0.1:12345
else:
    print "You provided no SEED_URI. You should call like [market myip seeduri]"

# Connection to one peer
class PeerConnection(object):
    def __init__(self, address):
        self._address = address

    def create_socket(self):
        self._ctx = zmq.Context()
        self._socket = self._ctx.socket(zmq.REQ)
        self._socket.connect(self._address)

    def cleanup_socket(self):
        self._socket.close()

    def send(self, data):
        self.send_raw(json.dumps(data))

    def send_raw(self, serialized):
        Process(target=self._send_raw, args=(serialized,)).start()

    def _send_raw(self, serialized):
        self.create_socket()

        self._socket.send(serialized)
        msg = self._socket.recv()
        self.on_message(msg)

        self.cleanup_socket()

    def on_message(self, msg):
        print "message received!", msg

    def closed(self, *args):
        print " - peer disconnected"

# Transport layer manages a list of peers
class TransportLayer(object):
    def __init__(self, port=DEFAULT_PORT):
        self._peers = {}
        self._callbacks = defaultdict(list)
        self._id = MY_IP[-1] # hack for logging
        self._port = port
        self._uri = 'tcp://%s:%s' % (MY_IP, self._port)

    def add_callback(self, section, callback):
        self._callbacks[section].append(callback)

    def trigger_callbacks(self, section, *data):
        for cb in self._callbacks[section]:
            cb(*data)
        if not section == 'all':
            for cb in self._callbacks['all']:
                cb(*data)

    def get_profile(self):
        return {'type': 'hello', 'uri': 'tcp://%s:12345' % MY_IP}

    def join_network(self):
        self.listen()

        if SEED_URI:
            self.init_peer({'uri': SEED_URI})

    def listen(self):
        Thread(target=self._listen).start()

    def _listen(self):
        self.log("init server %s %s" % (MY_IP, self._port))
        self._ctx = zmq.Context()
        self._socket = self._ctx.socket(zmq.REP)

        if MY_IP.startswith("127.0.0."):
            # we are in local test mode so bind that socket on the
            # specified IP
            self._socket.bind(self._uri)
        else:
            self._socket.bind('tcp://*:%s' % self._port)

        while True:
            message = self._socket.recv()
            self.on_raw_message(message)
            self._socket.send(json.dumps({'type': "ok"}))

    def closed(self, *args):
        print "client left"

    def _init_peer(self, msg):
        uri = msg['uri']
        self.log("init peer %s" %  msg)
        if not uri in self._peers:
            self._peers[uri] = PeerConnection(uri)

    def log(self, msg, pointer='-'):
        print " %s [%s] %s" % (pointer, self._id, msg)
        sys.stdout.flush()

    def send(self, data, send_to=None):

        #self.log("Data sent to p2p: %s" % data);

        # directed message
        if send_to:
            for peer in self._peers.values():
                if peer._pub == send_to:
                    if peer.send(data):
                        print 'Success'
                    else:
                        print 'Failed'
                    return
            print "Peer not found!", send_to, self._myself.get_pubkey()
            return

        # broadcast
        for peer in self._peers.values():
            try:
                if peer._pub:
                    peer.send(data)
                else:
                    serialized = json.dumps(data)
                    peer.send_raw(serialized)
            except:
                print "Error sending over peer!"
                traceback.print_exc()

    def on_message(self, msg):
        # here goes the application callbacks
        # we get a "clean" msg which is a dict holding whatever
        self.log("Data received: %s" % msg)
        self.trigger_callbacks(msg.get('type'), msg)

    def on_raw_message(self, serialized):
        self.log("connected " +str(len(serialized)))
        try:
            msg = json.loads(serialized[0])
        except:
            self.log("incorrect msg! " + serialized)
            return

        msg_type = msg.get('type')
        if msg_type == 'hello' and msg.get('uri'):
            self.init_peer(msg)
        else:
            self.on_message(msg)

<commit_msg>Removes requirement to specify seed uri.<commit_after>import sys
import json
from collections import defaultdict

import pyelliptic as ec

from zmq.eventloop import ioloop, zmqstream
import zmq
from multiprocessing import Process
from threading import Thread
ioloop.install()
import traceback

# Default port
DEFAULT_PORT=12345

# Get some command line pars
SEED_URI = False

if len(sys.argv) > 2:
    MY_IP = sys.argv[2]
else:
    MY_IP = "127.0.0.1"
if len(sys.argv) > 3:
    SEED_URI = sys.argv[3] # like tcp://127.0.0.1:12345

# Connection to one peer
class PeerConnection(object):
    def __init__(self, address):
        self._address = address

    def create_socket(self):
        self._ctx = zmq.Context()
        self._socket = self._ctx.socket(zmq.REQ)
        self._socket.connect(self._address)

    def cleanup_socket(self):
        self._socket.close()

    def send(self, data):
        self.send_raw(json.dumps(data))

    def send_raw(self, serialized):
        Process(target=self._send_raw, args=(serialized,)).start()

    def _send_raw(self, serialized):
        self.create_socket()

        self._socket.send(serialized)
        msg = self._socket.recv()
        self.on_message(msg)

        self.cleanup_socket()

    def on_message(self, msg):
        print "message received!", msg

    def closed(self, *args):
        print " - peer disconnected"

# Transport layer manages a list of peers
class TransportLayer(object):
    def __init__(self, port=DEFAULT_PORT):
        self._peers = {}
        self._callbacks = defaultdict(list)
        self._id = MY_IP[-1] # hack for logging
        self._port = port
        self._uri = 'tcp://%s:%s' % (MY_IP, self._port)

    def add_callback(self, section, callback):
        self._callbacks[section].append(callback)

    def trigger_callbacks(self, section, *data):
        for cb in self._callbacks[section]:
            cb(*data)
        if not section == 'all':
            for cb in self._callbacks['all']:
                cb(*data)

    def get_profile(self):
        return {'type': 'hello', 'uri': 'tcp://%s:12345' % MY_IP}

    def join_network(self):
        self.listen()

        if SEED_URI:
            self.init_peer({'uri': SEED_URI})

    def listen(self):
        Thread(target=self._listen).start()

    def _listen(self):
        self.log("init server %s %s" % (MY_IP, self._port))
        self._ctx = zmq.Context()
        self._socket = self._ctx.socket(zmq.REP)

        if MY_IP.startswith("127.0.0."):
            # we are in local test mode so bind that socket on the
            # specified IP
            self._socket.bind(self._uri)
        else:
            self._socket.bind('tcp://*:%s' % self._port)

        while True:
            message = self._socket.recv()
            self.on_raw_message(message)
            self._socket.send(json.dumps({'type': "ok"}))

    def closed(self, *args):
        print "client left"

    def _init_peer(self, msg):
        uri = msg['uri']
        self.log("init peer %s" %  msg)
        if not uri in self._peers:
            self._peers[uri] = PeerConnection(uri)

    def log(self, msg, pointer='-'):
        print " %s [%s] %s" % (pointer, self._id, msg)
        sys.stdout.flush()

    def send(self, data, send_to=None):

        #self.log("Data sent to p2p: %s" % data);

        # directed message
        if send_to:
            for peer in self._peers.values():
                if peer._pub == send_to:
                    if peer.send(data):
                        print 'Success'
                    else:
                        print 'Failed'
                    return
            print "Peer not found!", send_to, self._myself.get_pubkey()
            return

        # broadcast
        for peer in self._peers.values():
            try:
                if peer._pub:
                    peer.send(data)
                else:
                    serialized = json.dumps(data)
                    peer.send_raw(serialized)
            except:
                print "Error sending over peer!"
                traceback.print_exc()

    def on_message(self, msg):
        # here goes the application callbacks
        # we get a "clean" msg which is a dict holding whatever
        self.log("Data received: %s" % msg)
        self.trigger_callbacks(msg.get('type'), msg)

    def on_raw_message(self, serialized):
        self.log("connected " +str(len(serialized)))
        try:
            msg = json.loads(serialized[0])
        except:
            self.log("incorrect msg! " + serialized)
            return

        msg_type = msg.get('type')
        if msg_type == 'hello' and msg.get('uri'):
            self.init_peer(msg)
        else:
            self.on_message(msg)

<|endoftext|>


<commit_before>import logging
from datetime import datetime

from django.conf import settings
from django.contrib.gis.geos import Point
from django.utils.timezone import make_aware

from geotrek.feedback.models import (
    AttachedMessage,
    MessageAttachedDocument,
    Report,
    ReportActivity,
    ReportAttachedDocument,
    ReportCategory,
    ReportProblemMagnitude,
    ReportStatus,
)

from .helpers import SuricateRequestManager

logger = logging.getLogger(__name__)

class SuricateParser(SuricateRequestManager):
    def parse_date(self, date):
        """Parse datetime string from Suricate Rest API"""
        date_no_timezone = datetime.strptime(date, "%B, %d %Y %H:%M:%S")
        date_timezone = make_aware(date_no_timezone)
        return date_timezone

    def get_activities(self):
        """Get activities list from Suricate Rest API"""

        data = self.get_from_suricate("wsGetActivities")

        # Parse activities and create
        for activity in data["activites"]:
            obj, updated = ReportActivity.objects.update_or_create(
                suricate_id=activity["id"], defaults={"label": activity["libelle"]}
            )
            if updated:
                logger.info(
                    f"New or updated activity - id: {activity['id']}, label: {activity['libelle']}"
                )

    def get_statuses(self):
        """Get statuses list from Suricate Rest API"""

        data = self.get_from_suricate("wsGetStatusList")

        # Parse statuses and create
        for status in data["statuts"]:
            obj, updated = ReportStatus.objects.get_or_create(
                suricate_id=status["id"], defaults={"label": status["libelle"]}
            )
            if updated:
                logger.info(
                    f"New or updated status - id: {status['id']}, label: {status['libelle']}"
                )

    def get_alerts(self):
        """Get reports list from Suricate Rest API"""

        data = self.get_from_suricate("wsGetAlerts")
        # i = 0
        # Parse alerts
        for report in data["alertes"]:
            # print("Alert " + str(i))

            # Parse dates
            rep_updated = self.parse_date(report["updated"])
            rep_creation = self.parse_date(report["datedepot"])

            # Parse geom
            rep_gps = Point(report["gpslongitude"], report["gpslatitude"], srid=4326)
            rep_srid = rep_gps.transform(settings.SRID, clone=True)
            rep_point = Point(rep_srid.coords)

            # Parse magnitude
            rep_magnitude, created = ReportProblemMagnitude.objects.get_or_create(
                label=report["ampleur"]
            )
            if created:
                logger.info(
                    f"Created new feedback magnitude - label: {report['ampleur']}"
                )

            # Parse category
            rep_category, created = ReportCategory.objects.get_or_create(
                label=report["type"]
            )
            if created:
                logger.info(f"Created new feedback category - label: {report['type']}")

            # Parse status
            rep_status = ReportStatus.objects.get(suricate_id=report["statut"])

            # Parse activity
            rep_activity = ReportActivity.objects.get(suricate_id=report["idactivite"])
            if created:
                logger.info(f"Created new feedback category - label: {report['type']}")

            # Create report object
            fields = {
                "locked": bool(report["locked"]),
                "email": report["emaildeposant"],
                "comment": report["commentaire"],
                "geom": rep_point,
                "origin": report["origin"],
                "activity": rep_activity,
                "category": rep_category,
                "problem_magnitude": rep_magnitude,
                "status": rep_status,
                "created": rep_creation,
                "last_updated": rep_updated,
            }
            report_obj, updated = Report.objects.update_or_create(
                uid=report["uid"], defaults=fields
            )
            if updated:
                logger.info(
                    f"New or updated report - id: {report['uid']}, location: {report_obj.geom}"
                )

            # Parse documents attached to report
            self.create_documents(report["documents"], report_obj, "Report")

            # Parse messages attached to report
            self.create_messages(report["messages"], report_obj)
            # i += 1

    def create_documents(self, documents, parent, type_parent):
        """Parse documents list from Suricate Rest API"""

        for document in documents:
            # Parse fields
            fields = {"file_name": document["nomfichier"], "url": document["url"]}

            # Attach document to the right object
            if type_parent == "Report":
                # Attach document to a report and create
                fields["report"] = parent
                document_obj, updated = ReportAttachedDocument.objects.update_or_create(
                    suricate_id=document["id"], defaults=fields
                )
                if updated:
                    logger.info(
                        f"New or updated document - id: {document['id']}, parent: {parent.uid}"
                    )
            elif type_parent == "Message":
                # Attach document to a message and create
                fields["message"] = parent
                (
                    document_obj,
                    updated,
                ) = MessageAttachedDocument.objects.update_or_create(
                    suricate_id=document["id"], defaults=fields
                )
                if updated:
                    logger.info(
                        f"New or updated Message document - id: {document['id']}, parent: {parent.uid}"
                    )

    def create_messages(self, messages, parent):
        """Parse messages list from Suricate Rest API"""

        for message in messages:
            # Parse date
            msg_creation = self.parse_date(message["date"])

            # Parse fields
            fields = {
                "date": msg_creation,
                "author": message["redacteur"],
                "content": message["texte"],
                "type": message["type"],
                "suricate_id": message["id"],
                "report": parent,
            }

            # Create message object
            message_obj, updated = AttachedMessage.objects.update_or_create(
                suricate_id=message["id"], defaults=fields
            )
            if updated:
                logger.info(
                    f"New or updated Message - id: {message['id']}, parent: {parent.uid}"
                )

            # Parse documents attached to message
            self.create_documents(message["documents"], message_obj, "Message")
<commit_msg>Fix log message in documents parsing<commit_after>import logging
from datetime import datetime

from django.conf import settings
from django.contrib.gis.geos import Point
from django.utils.timezone import make_aware

from geotrek.feedback.models import (
    AttachedMessage,
    MessageAttachedDocument,
    Report,
    ReportActivity,
    ReportAttachedDocument,
    ReportCategory,
    ReportProblemMagnitude,
    ReportStatus,
)

from .helpers import SuricateRequestManager

logger = logging.getLogger(__name__)

class SuricateParser(SuricateRequestManager):
    def parse_date(self, date):
        """Parse datetime string from Suricate Rest API"""
        date_no_timezone = datetime.strptime(date, "%B, %d %Y %H:%M:%S")
        date_timezone = make_aware(date_no_timezone)
        return date_timezone

    def get_activities(self):
        """Get activities list from Suricate Rest API"""

        data = self.get_from_suricate("wsGetActivities")

        # Parse activities and create
        for activity in data["activites"]:
            obj, updated = ReportActivity.objects.update_or_create(
                suricate_id=activity["id"], defaults={"label": activity["libelle"]}
            )
            if updated:
                logger.info(
                    f"New or updated activity - id: {activity['id']}, label: {activity['libelle']}"
                )

    def get_statuses(self):
        """Get statuses list from Suricate Rest API"""

        data = self.get_from_suricate("wsGetStatusList")

        # Parse statuses and create
        for status in data["statuts"]:
            obj, updated = ReportStatus.objects.get_or_create(
                suricate_id=status["id"], defaults={"label": status["libelle"]}
            )
            if updated:
                logger.info(
                    f"New or updated status - id: {status['id']}, label: {status['libelle']}"
                )

    def get_alerts(self):
        """Get reports list from Suricate Rest API"""

        data = self.get_from_suricate("wsGetAlerts")
        # i = 0
        # Parse alerts
        for report in data["alertes"]:
            # print("Alert " + str(i))

            # Parse dates
            rep_updated = self.parse_date(report["updated"])
            rep_creation = self.parse_date(report["datedepot"])

            # Parse geom
            rep_gps = Point(report["gpslongitude"], report["gpslatitude"], srid=4326)
            rep_srid = rep_gps.transform(settings.SRID, clone=True)
            rep_point = Point(rep_srid.coords)

            # Parse magnitude
            rep_magnitude, created = ReportProblemMagnitude.objects.get_or_create(
                label=report["ampleur"]
            )
            if created:
                logger.info(
                    f"Created new feedback magnitude - label: {report['ampleur']}"
                )

            # Parse category
            rep_category, created = ReportCategory.objects.get_or_create(
                label=report["type"]
            )
            if created:
                logger.info(f"Created new feedback category - label: {report['type']}")

            # Parse status
            rep_status = ReportStatus.objects.get(suricate_id=report["statut"])

            # Parse activity
            rep_activity = ReportActivity.objects.get(suricate_id=report["idactivite"])
            if created:
                logger.info(f"Created new feedback category - label: {report['type']}")

            # Create report object
            fields = {
                "locked": bool(report["locked"]),
                "email": report["emaildeposant"],
                "comment": report["commentaire"],
                "geom": rep_point,
                "origin": report["origin"],
                "activity": rep_activity,
                "category": rep_category,
                "problem_magnitude": rep_magnitude,
                "status": rep_status,
                "created": rep_creation,
                "last_updated": rep_updated,
            }
            report_obj, updated = Report.objects.update_or_create(
                uid=report["uid"], defaults=fields
            )
            if updated:
                logger.info(
                    f"New or updated report - id: {report['uid']}, location: {report_obj.geom}"
                )

            # Parse documents attached to report
            self.create_documents(report["documents"], report_obj, "Report")

            # Parse messages attached to report
            self.create_messages(report["messages"], report_obj)
            # i += 1

    def create_documents(self, documents, parent, type_parent):
        """Parse documents list from Suricate Rest API"""

        for document in documents:
            # Parse fields
            fields = {"file_name": document["nomfichier"], "url": document["url"]}

            # Attach document to the right object
            if type_parent == "Report":
                # Attach document to a report and create
                fields["report"] = parent
                document_obj, updated = ReportAttachedDocument.objects.update_or_create(
                    suricate_id=document["id"], defaults=fields
                )
                if updated:
                    logger.info(
                        f"New or updated document - id: {document['id']}, parent: {parent.uid}"
                    )
            elif type_parent == "Message":
                # Attach document to a message and create
                fields["message"] = parent
                (
                    document_obj,
                    updated,
                ) = MessageAttachedDocument.objects.update_or_create(
                    suricate_id=document["id"], defaults=fields
                )
                if updated:
                    logger.info(
                        f"New or updated Message document - id: {document['id']}, parent: {parent.suricate_id}"
                    )

    def create_messages(self, messages, parent):
        """Parse messages list from Suricate Rest API"""

        for message in messages:
            # Parse date
            msg_creation = self.parse_date(message["date"])

            # Parse fields
            fields = {
                "date": msg_creation,
                "author": message["redacteur"],
                "content": message["texte"],
                "type": message["type"],
                "suricate_id": message["id"],
                "report": parent,
            }

            # Create message object
            message_obj, updated = AttachedMessage.objects.update_or_create(
                suricate_id=message["id"], defaults=fields
            )
            if updated:
                logger.info(
                    f"New or updated Message - id: {message['id']}, parent: {parent.uid}"
                )

            # Parse documents attached to message
            self.create_documents(message["documents"], message_obj, "Message")
<|endoftext|>


<commit_before>import os

def test_notebooks():
    notebooks = ['Nearest_Neighbors_and_Gram_Matrix.ipynb']
    problems = []
    for notebook in notebooks:
        cmd = 'wget https://raw.githubusercontent.com/fabriziocosta/EDeN_examples/master/%s' % notebook
        res = os.system(cmd)
        cmd = 'jupyter nbconvert --to=html --ExecutePreprocessor.enabled=True --ExecutePreprocessor.timeout=300 %s' % notebook
        res = os.system(cmd)
        os.system('rm -f %s' % notebook)

        assert res == 0

test_notebooks()

<commit_msg>fix nb test<commit_after>import os

def test_notebooks():
    notebooks = ['Nearest_Neighbors_and_Gram_Matrix.ipynb']
    problems = []
    for notebook in notebooks:
        cmd = 'wget https://raw.githubusercontent.com/fabriziocosta/EDeN_examples/master/%s' % notebook
        res = os.system(cmd)
        cmd = 'jupyter nbconvert --to=html --ExecutePreprocessor.enabled=True --ExecutePreprocessor.timeout=300 %s' % notebook
        res = os.system(cmd)
        os.system('rm -f %s' % notebook)

        assert res == 0

<|endoftext|>


<commit_before>from collections import OrderedDict

from django.core.exceptions import ObjectDoesNotExist

from cbmonitor import models

class Report(object):

    def __new__(cls, snapshots, report_type):
        try:
            return eval(report_type)(snapshots)
        except (NameError, SyntaxError):
            raise NotImplementedError("Unknown report type")

class BaseReport(object):

    metrics = OrderedDict((
        ("xdcr_lag", [
            "xdcr_lag",
            "xdcr_persistence_time",
            "xdcr_diff",
        ]),
        ("spring_query_latency", [
            "latency_query",
        ]),
        ("spring_tuq_latency", [
            "latency_query",
            "latency_tuq",
        ]),
        ("spring_latency", [
            "latency_set",
            "latency_get",
        ]),
        ("sync_gateway", [
            "Sys",
            "Alloc",
            "HeapAlloc",
            "HeapObjects",
            "PauseNs",
            "PauseTotalNs",
            "PausesPct",
            "NumGC",
        ]),
        ("ns_server", [
            "couch_views_ops",
            "ops",
            "cmd_get",
            "cmd_set",
            "delete_hits",
            "cas_hits",
            "xdc_ops",
            "ep_num_ops_get_meta",
            "ep_num_ops_set_meta",
            "ep_num_ops_del_meta",
            "replication_changes_left",
            "replication_size_rep_queue",
            "replication_rate_replication",
            "replication_bandwidth_usage",
            "replication_work_time",
            "replication_commit_time",
            "replication_active_vbreps",
            "replication_waiting_vbreps",
            "replication_num_checkpoints",
            "replication_num_failedckpts",
            "replication_meta_latency_wt",
            "replication_docs_latency_wt",
            "curr_connections",
            "hibernated_waked",
            "curr_items",
            "mem_used",
            "ep_meta_data_memory",
            "vb_active_resident_items_ratio",
            "vb_replica_resident_items_ratio",
            "ep_num_value_ejects",
            "ep_tmp_oom_errors",
            "disk_write_queue",
            "ep_cache_miss_rate",
            "ep_bg_fetched",
            "ep_diskqueue_drain",
            "avg_bg_wait_time",
            "avg_disk_commit_time",
            "avg_disk_update_time",
            "couch_docs_data_size",
            "couch_docs_actual_disk_size",
            "couch_views_data_size",
            "couch_views_actual_disk_size",
            "couch_total_disk_size",
            "couch_docs_fragmentation",
            "couch_views_fragmentation",
        ]),
        ("atop", [
            "sync_gateway_rss",
            "sync_gateway_cpu",
            "beam.smp_rss",
            "beam.smp_cpu",
            "memcached_cpu",
        ]),
        ("iostat", [
            "data_rbps",
            "data_wbps",
            "data_avgqusz",
            "data_util",
            "index_rbps",
            "index_wbps",
            "index_avgqusz",
            "index_util",
        ]),
    ))

    def __init__(self, snapshots):
        self.snapshots = snapshots
        clusters = [cluster for snapshot, cluster in snapshots]
        self.buckets = models.Bucket.objects.filter(cluster=clusters[0])
        self.servers = models.Server.objects.filter(cluster=clusters[0])

    def merge_metrics(self):
        base_metrics = BaseReport.metrics
        for collector in set(base_metrics) & set(self.metrics):
            self.metrics[collector] += base_metrics[collector]
        self.metrics = OrderedDict(base_metrics, **self.metrics)

    def __iter__(self):
        for bucket in self.buckets:
            observables = []
            for snapshot, cluster in self.snapshots:
                _bucket = models.Bucket.objects.get(
                    cluster=cluster,
                    name=bucket.name
                )
                try:
                    observable = models.Observable.objects.get(
                        cluster=cluster,
                        collector="active_tasks",
                        name="bucket_compaction_progress",
                        server__isnull=True,
                        bucket=_bucket,
                    )
                    observables.append((observable, snapshot))
                except ObjectDoesNotExist:
                    pass
            if observables:
                yield observables

        for collector, metrics in self.metrics.iteritems():
            for metric in metrics:
                for bucket in self.buckets:
                    observables = []
                    for snapshot, cluster in self.snapshots:
                        try:
                            _bucket = models.Bucket.objects.get(
                                cluster=cluster,
                                name=bucket.name
                            )
                            observable = models.Observable.objects.get(
                                cluster=cluster,
                                collector=collector,
                                name=metric,
                                server__isnull=True,
                                bucket=_bucket,
                            )
                            observables.append((observable, snapshot))
                        except ObjectDoesNotExist:
                            pass
                    if observables:
                        yield observables
            if collector in ("atop", "iostat", "sync_gateway"):
                for metric in metrics:
                    for server in self.servers:
                        observables = []
                        for snapshot, cluster in self.snapshots:
                            try:
                                _server = models.Server.objects.get(
                                    cluster=cluster,
                                    address=server.address
                                )
                                observable = models.Observable.objects.get(
                                    cluster=cluster,
                                    collector=collector,
                                    name=metric,
                                    server=_server,
                                    bucket__isnull=True,
                                )
                                observables.append((observable, snapshot))
                            except ObjectDoesNotExist:
                                pass
                        if observables:
                            yield observables

class BaseRebalanceReport(BaseReport):

    def __iter__(self):
        observables = []
        for snapshot, cluster in self.snapshots:
            try:
                observable = models.Observable.objects.get(
                    cluster=cluster,
                    collector="active_tasks",
                    name="rebalance_progress",
                    server__isnull=True,
                    bucket__isnull=True,
                )
                observables.append((observable, snapshot))
            except ObjectDoesNotExist:
                pass
        if observables:
            yield observables
        for observables in super(BaseRebalanceReport, self).__iter__():
            yield observables

class BaseXdcrReport(BaseReport):

    pass

class BaseKVReport(BaseReport):

    pass

class BaseViewsReport(BaseReport):

    pass

class BaseTuqReport(BaseReport):

    pass

class BaseRebalanceViewsReport(BaseRebalanceReport):

    pass

class BaseRebalanceXdcrReport(BaseRebalanceReport):

    pass

class SyncGatewayReport(BaseReport):

    pass
<commit_msg>avoid unnecessary db queries<commit_after>from collections import OrderedDict

from django.core.exceptions import ObjectDoesNotExist

from cbmonitor import models

class Report(object):

    def __new__(cls, snapshots, report_type):
        try:
            return eval(report_type)(snapshots)
        except (NameError, SyntaxError):
            raise NotImplementedError("Unknown report type")

class BaseReport(object):

    metrics = OrderedDict((
        ("xdcr_lag", [
            "xdcr_lag",
            "xdcr_persistence_time",
            "xdcr_diff",
        ]),
        ("spring_query_latency", [
            "latency_query",
        ]),
        ("spring_tuq_latency", [
            "latency_query",
            "latency_tuq",
        ]),
        ("spring_latency", [
            "latency_set",
            "latency_get",
        ]),
        ("sync_gateway", [
            "Sys",
            "Alloc",
            "HeapAlloc",
            "HeapObjects",
            "PauseNs",
            "PauseTotalNs",
            "PausesPct",
            "NumGC",
        ]),
        ("ns_server", [
            "couch_views_ops",
            "ops",
            "cmd_get",
            "cmd_set",
            "delete_hits",
            "cas_hits",
            "xdc_ops",
            "ep_num_ops_get_meta",
            "ep_num_ops_set_meta",
            "ep_num_ops_del_meta",
            "replication_changes_left",
            "replication_size_rep_queue",
            "replication_rate_replication",
            "replication_bandwidth_usage",
            "replication_work_time",
            "replication_commit_time",
            "replication_active_vbreps",
            "replication_waiting_vbreps",
            "replication_num_checkpoints",
            "replication_num_failedckpts",
            "replication_meta_latency_wt",
            "replication_docs_latency_wt",
            "curr_connections",
            "hibernated_waked",
            "curr_items",
            "mem_used",
            "ep_meta_data_memory",
            "vb_active_resident_items_ratio",
            "vb_replica_resident_items_ratio",
            "ep_num_value_ejects",
            "ep_tmp_oom_errors",
            "disk_write_queue",
            "ep_cache_miss_rate",
            "ep_bg_fetched",
            "ep_diskqueue_drain",
            "avg_bg_wait_time",
            "avg_disk_commit_time",
            "avg_disk_update_time",
            "couch_docs_data_size",
            "couch_docs_actual_disk_size",
            "couch_views_data_size",
            "couch_views_actual_disk_size",
            "couch_total_disk_size",
            "couch_docs_fragmentation",
            "couch_views_fragmentation",
        ]),
        ("atop", [
            "sync_gateway_rss",
            "sync_gateway_cpu",
            "beam.smp_rss",
            "beam.smp_cpu",
            "memcached_cpu",
        ]),
        ("iostat", [
            "data_rbps",
            "data_wbps",
            "data_avgqusz",
            "data_util",
            "index_rbps",
            "index_wbps",
            "index_avgqusz",
            "index_util",
        ]),
    ))

    def __init__(self, snapshots):
        self.snapshots = snapshots
        clusters = [cluster for snapshot, cluster in snapshots]
        self.buckets = models.Bucket.objects.filter(cluster=clusters[0])
        self.servers = models.Server.objects.filter(cluster=clusters[0])

    def merge_metrics(self):
        base_metrics = BaseReport.metrics
        for collector in set(base_metrics) & set(self.metrics):
            self.metrics[collector] += base_metrics[collector]
        self.metrics = OrderedDict(base_metrics, **self.metrics)

    def __iter__(self):
        for bucket in self.buckets:
            observables = []
            for snapshot, cluster in self.snapshots:
                _bucket = models.Bucket.objects.get(
                    cluster=cluster,
                    name=bucket.name
                )
                try:
                    observable = models.Observable.objects.get(
                        cluster=cluster,
                        collector="active_tasks",
                        name="bucket_compaction_progress",
                        server__isnull=True,
                        bucket=_bucket,
                    )
                    observables.append((observable, snapshot))
                except ObjectDoesNotExist:
                    pass
            if observables:
                yield observables

        for collector, metrics in self.metrics.iteritems():
            if collector in ("atop", "iostat", "sync_gateway"):
                for metric in metrics:
                    for server in self.servers:
                        observables = []
                        for snapshot, cluster in self.snapshots:
                            try:
                                _server = models.Server.objects.get(
                                    cluster=cluster,
                                    address=server.address
                                )
                                observable = models.Observable.objects.get(
                                    cluster=cluster,
                                    collector=collector,
                                    name=metric,
                                    server=_server,
                                    bucket__isnull=True,
                                )
                                observables.append((observable, snapshot))
                            except ObjectDoesNotExist:
                                pass
                        if observables:
                            yield observables
            else:
                for metric in metrics:
                    for bucket in self.buckets:
                        observables = []
                        for snapshot, cluster in self.snapshots:
                            try:
                                _bucket = models.Bucket.objects.get(
                                    cluster=cluster,
                                    name=bucket.name
                                )
                                observable = models.Observable.objects.get(
                                    cluster=cluster,
                                    collector=collector,
                                    name=metric,
                                    server__isnull=True,
                                    bucket=_bucket,
                                )
                                observables.append((observable, snapshot))
                            except ObjectDoesNotExist:
                                pass
                        if observables:
                            yield observables

class BaseRebalanceReport(BaseReport):

    def __iter__(self):
        observables = []
        for snapshot, cluster in self.snapshots:
            try:
                observable = models.Observable.objects.get(
                    cluster=cluster,
                    collector="active_tasks",
                    name="rebalance_progress",
                    server__isnull=True,
                    bucket__isnull=True,
                )
                observables.append((observable, snapshot))
            except ObjectDoesNotExist:
                pass
        if observables:
            yield observables
        for observables in super(BaseRebalanceReport, self).__iter__():
            yield observables

class BaseXdcrReport(BaseReport):

    pass

class BaseKVReport(BaseReport):

    pass

class BaseViewsReport(BaseReport):

    pass

class BaseTuqReport(BaseReport):

    pass

class BaseRebalanceViewsReport(BaseRebalanceReport):

    pass

class BaseRebalanceXdcrReport(BaseRebalanceReport):

    pass

class SyncGatewayReport(BaseReport):

    pass
<|endoftext|>


<commit_before>#!/usr/bin/env python

import random
import time
import unittest
from queue import Queue
from unittest.mock import patch

from netils import build_byte_string

from tests.unit.faucet.test_valve import ValveTestBases

DOT1X_DP1_CONFIG = """
        dp_id: 1
        dot1x:
            nfv_intf: lo
            radius_ip: 127.0.0.1
            radius_port: 1812
            radius_secret: SECRET"""

DOT1X_CONFIG = """
acls:
    eapol_to_nfv:
        - rule:
            dl_type: 0x888e
            actions:
                output:
                    # set_fields:
                        # - eth_dst: NFV_MAC
                    port: p2
        - rule:
            eth_src: ff:ff:ff:ff:ff:ff
            actions:
                allow: 0
        - rule:
            actions:
                allow: 0
    eapol_from_nfv:
        - rule:
            dl_type: 0x888e
            # eth_dst: NFV_MAC
            actions:
                output:
                    # set_fields:
                        # - eth_dst: 01:80:c2:00:00:03
                    port: p1
        - rule:
            actions:
                allow: 0
    allowall:
        - rule:
            actions:
                allow: 1
dps:
    s1:
        hardware: 'GenericTFM'
%s
        interfaces:
            p1:
                number: 1
                native_vlan: v100
                dot1x: True
                acl_in: eapol_to_nfv
            p2:
                number: 2
                native_vlan: v100
                acl_in: eapol_from_nfv
            p3:
                number: 3
                native_vlan: v100
                acl_in: allowall
vlans:
    v100:
        vid: 0x100
""" % DOT1X_DP1_CONFIG

FROM_SUPPLICANT = Queue()
TO_SUPPLICANT = Queue()
FROM_RADIUS = Queue()
TO_RADIUS = Queue()

def supplicant_replies():
    header = "0000000000010242ac17006f888e"
    replies = [build_byte_string(header + "01000009027400090175736572"),
               build_byte_string(header + "010000160275001604103abcadc86714b2d75d09dd7ff53edf6b")]

    for r in replies:
        yield r

def radius_replies():
    replies = [build_byte_string("0b040050e5e40d846576a2310755e906c4b2b5064f180175001604101a16a3baa37a0238f33384f6c11067425012ce61ba97026b7a05b194a930a922405218126aa866456add628e3a55a4737872cad6"),
               build_byte_string("02050032fb4c4926caa21a02f74501a65c96f9c74f06037500045012c060ca6a19c47d0998c7b20fd4d771c1010675736572")]
    for r in replies:
        yield r

def urandom():
    _list = [b'\x87\xf5[\xa71\xeeOA;}\\t\xde\xd7.=',
             b'\xf7\xe0\xaf\xc7Q!\xa2\xa9\xa3\x8d\xf7\xc6\x85\xa8k\x06']
    for l in _list:
        yield l

URANDOM_GENERATOR = urandom()

def urandom_helper(size):
    return next(URANDOM_GENERATOR)

SUPPLICANT_REPLY_GENERATOR = supplicant_replies()
RADIUS_REPLY_GENERATOR = radius_replies()

def eap_receive(chewie):
    return FROM_SUPPLICANT.get()

def eap_send(chewie, data):
    TO_SUPPLICANT.put(data)
    try:
        n = next(SUPPLICANT_REPLY_GENERATOR)
    except StopIteration:
        return
    if n:
        FROM_SUPPLICANT.put(n)

def radius_receive(chewie):
    return FROM_RADIUS.get()

def radius_send(chewie, data):
    TO_RADIUS.put(data)
    try:
        n = next(RADIUS_REPLY_GENERATOR)
    except StopIteration:
        return
    if n:
        FROM_RADIUS.put(n)

def nextId(eap_sm):  # pylint: disable=invalid-name
    """Determines the next identifier value to use, based on the previous one.
    Returns:
        integer"""
    if eap_sm.currentId is None:
        # I'm assuming we cant have ids wrap around in the same series.
        #  so the 200 provides a large buffer.
        return 116
    _id = eap_sm.currentId + 1
    if _id > 255:
        return random.randint(0, 200)
    return _id

def get_next_radius_packet_id(chewie):
    """Calulate the next RADIUS Packet ID
    Returns:
        int
    """
    if chewie.radius_id == -1:
        chewie.radius_id = 4
        return chewie.radius_id
    chewie.radius_id += 1
    if chewie.radius_id > 255:
        chewie.radius_id = 0
    return chewie.radius_id

class FaucetDot1XTest(ValveTestBases.ValveTestSmall):
    """Test chewie api"""

    def setUp(self):
        # self.mock_socket = mock
        FROM_SUPPLICANT.put(build_byte_string("0000000000010242ac17006f888e01010000"))
        self.setup_valve(DOT1X_CONFIG)

    @patch('faucet.faucet_dot1x.chewie.os.urandom', urandom_helper)
    @patch('faucet.faucet_dot1x.chewie.FullEAPStateMachine.nextId', nextId)
    @patch('faucet.faucet_dot1x.chewie.Chewie.get_next_radius_packet_id', get_next_radius_packet_id)
    @patch('faucet.faucet_dot1x.chewie.Chewie.radius_send', radius_send)
    @patch('faucet.faucet_dot1x.chewie.Chewie.radius_receive', radius_receive)
    @patch('faucet.faucet_dot1x.chewie.Chewie.eap_send', eap_send)
    @patch('faucet.faucet_dot1x.chewie.Chewie.eap_receive', eap_receive)
    def test_success_dot1x(self):
        """Test success api"""
        self.dot1x.reset(valves=self.valves_manager.valves)
        time.sleep(40)

    def tearDown(self):
        with open('%s/faucet.log' % self.tmpdir, 'r') as log:
            print(log.read())
        super(FaucetDot1XTest, self).tearDown()

if __name__ == "__main__":
    unittest.main()  # pytype: disable=module-attr
<commit_msg>check flows added.<commit_after>#!/usr/bin/env python

import random
import time
import unittest
from queue import Queue
from unittest.mock import patch

from netils import build_byte_string

from tests.unit.faucet.test_valve import ValveTestBases

DOT1X_DP1_CONFIG = """
        dp_id: 1
        dot1x:
            nfv_intf: lo
            radius_ip: 127.0.0.1
            radius_port: 1812
            radius_secret: SECRET"""

DOT1X_CONFIG = """
acls:
    eapol_to_nfv:
        - rule:
            dl_type: 0x888e
            actions:
                output:
                    # set_fields:
                        # - eth_dst: NFV_MAC
                    port: p2
        - rule:
            eth_src: ff:ff:ff:ff:ff:ff
            actions:
                allow: 0
        - rule:
            actions:
                allow: 0
    eapol_from_nfv:
        - rule:
            dl_type: 0x888e
            # eth_dst: NFV_MAC
            actions:
                output:
                    # set_fields:
                        # - eth_dst: 01:80:c2:00:00:03
                    port: p1
        - rule:
            actions:
                allow: 0
    allowall:
        - rule:
            actions:
                allow: 1
dps:
    s1:
        hardware: 'GenericTFM'
%s
        interfaces:
            p1:
                number: 1
                native_vlan: v100
                dot1x: True
                acl_in: eapol_to_nfv
            p2:
                number: 2
                native_vlan: v100
                acl_in: eapol_from_nfv
            p3:
                number: 3
                native_vlan: v100
                acl_in: allowall
vlans:
    v100:
        vid: 0x100
""" % DOT1X_DP1_CONFIG

FROM_SUPPLICANT = Queue()
TO_SUPPLICANT = Queue()
FROM_RADIUS = Queue()
TO_RADIUS = Queue()

def supplicant_replies():
    header = "0000000000010242ac17006f888e"
    replies = [build_byte_string(header + "01000009027400090175736572"),
               build_byte_string(header + "010000160275001604103abcadc86714b2d75d09dd7ff53edf6b")]

    for r in replies:
        yield r

def radius_replies():
    replies = [build_byte_string("0b040050e5e40d846576a2310755e906c4b2b5064f180175001604101a16a3baa37a0238f33384f6c11067425012ce61ba97026b7a05b194a930a922405218126aa866456add628e3a55a4737872cad6"),
               build_byte_string("02050032fb4c4926caa21a02f74501a65c96f9c74f06037500045012c060ca6a19c47d0998c7b20fd4d771c1010675736572")]
    for r in replies:
        yield r

def urandom():
    _list = [b'\x87\xf5[\xa71\xeeOA;}\\t\xde\xd7.=',
             b'\xf7\xe0\xaf\xc7Q!\xa2\xa9\xa3\x8d\xf7\xc6\x85\xa8k\x06']
    for l in _list:
        yield l

URANDOM_GENERATOR = urandom()

def urandom_helper(size):
    return next(URANDOM_GENERATOR)

SUPPLICANT_REPLY_GENERATOR = supplicant_replies()
RADIUS_REPLY_GENERATOR = radius_replies()

def eap_receive(chewie):
    return FROM_SUPPLICANT.get()

def eap_send(chewie, data):
    TO_SUPPLICANT.put(data)
    try:
        n = next(SUPPLICANT_REPLY_GENERATOR)
    except StopIteration:
        return
    if n:
        FROM_SUPPLICANT.put(n)

def radius_receive(chewie):
    return FROM_RADIUS.get()

def radius_send(chewie, data):
    TO_RADIUS.put(data)
    try:
        n = next(RADIUS_REPLY_GENERATOR)
    except StopIteration:
        return
    if n:
        FROM_RADIUS.put(n)

def nextId(eap_sm):  # pylint: disable=invalid-name
    """Determines the next identifier value to use, based on the previous one.
    Returns:
        integer"""
    if eap_sm.currentId is None:
        # I'm assuming we cant have ids wrap around in the same series.
        #  so the 200 provides a large buffer.
        return 116
    _id = eap_sm.currentId + 1
    if _id > 255:
        return random.randint(0, 200)
    return _id

def get_next_radius_packet_id(chewie):
    """Calulate the next RADIUS Packet ID
    Returns:
        int
    """
    if chewie.radius_id == -1:
        chewie.radius_id = 4
        return chewie.radius_id
    chewie.radius_id += 1
    if chewie.radius_id > 255:
        chewie.radius_id = 0
    return chewie.radius_id

class FaucetDot1XTest(ValveTestBases.ValveTestSmall):
    """Test chewie api"""

    def setUp(self):
        self.setup_valve(DOT1X_CONFIG)

    @patch('faucet.faucet_dot1x.chewie.os.urandom', urandom_helper)
    @patch('faucet.faucet_dot1x.chewie.FullEAPStateMachine.nextId', nextId)
    @patch('faucet.faucet_dot1x.chewie.Chewie.get_next_radius_packet_id', get_next_radius_packet_id)
    @patch('faucet.faucet_dot1x.chewie.Chewie.radius_send', radius_send)
    @patch('faucet.faucet_dot1x.chewie.Chewie.radius_receive', radius_receive)
    @patch('faucet.faucet_dot1x.chewie.Chewie.eap_send', eap_send)
    @patch('faucet.faucet_dot1x.chewie.Chewie.eap_receive', eap_receive)
    def test_success_dot1x(self):
        """Test success api"""
        self.dot1x.reset(valves=self.valves_manager.valves)

        FROM_SUPPLICANT.put(build_byte_string("0000000000010242ac17006f888e01010000"))
        time.sleep(5)
        with open('%s/faucet.log' % self.tmpdir, 'r') as log:
            for l in log.readlines():
                if 'Successful auth' in l:
                    break
            else:
                self.fail('Cannot find "Successful auth" string in faucet.log')
        self.assertEqual(2,
                         len(self.last_flows_to_dp[1]))

if __name__ == "__main__":
    unittest.main()  # pytype: disable=module-attr
<|endoftext|>


<commit_before>from mock import patch
try:
    import unittest2 as unittest
except ImportError:
    import unittest
from graphite.render.attime import parseTimeReference, parseATTime, parseTimeOffset, getUnitString

from datetime import datetime, timedelta

from django.utils import timezone
from django.test import TestCase
import pytz
import mock

class MockedDateTime(datetime):
    def __new__(cls, *args, **kwargs):
        return datetime.__new__(datetime, *args, **kwargs)
    @classmethod
    def now(cls):
        return cls(2015, 3, 8, 12, 0, 0)

@mock.patch('graphite.render.attime.datetime', MockedDateTime)
class ATTimeTimezoneTests(TestCase):
    default_tz = timezone.get_current_timezone()
    specified_tz = pytz.timezone("America/Los_Angeles")
    def test_should_return_absolute_time(self):
        time_string = '12:0020150308'

        expected_time = self.default_tz.localize(datetime.strptime(time_string,'%H:%M%Y%m%d'))
        actual_time = parseATTime(time_string)
        self.assertEqual(actual_time, expected_time)

    def test_absolute_time_should_respect_tz(self):
        time_string = '12:0020150308'
        expected_time = self.specified_tz.localize(datetime.strptime(time_string, '%H:%M%Y%m%d'))
        actual_time = parseATTime(time_string, self.specified_tz)
        self.assertEqual(actual_time, expected_time)

    def test_midnight(self):
        expected_time = self.specified_tz.localize(datetime.strptime("0:00_20150308", '%H:%M_%Y%m%d'))
        actual_time = parseATTime("midnight", self.specified_tz)
        self.assertEqual(actual_time, expected_time)

    def test_offset_with_tz(self):
        expected_time = self.specified_tz.localize(datetime.strptime("5:00_20150308", '%H:%M_%Y%m%d'))
        actual_time = parseATTime("midnight+5h", self.specified_tz)
        self.assertEqual(actual_time, expected_time)

    def test_relative_day_with_tz(self):
        expected_time = self.specified_tz.localize(datetime.strptime("0:00_20150309", '%H:%M_%Y%m%d'))
        actual_time = parseATTime("midnight_tomorrow", self.specified_tz)
        self.assertEqual(actual_time, expected_time)

    def test_relative_day_and_offset_with_tz(self):
        expected_time = self.specified_tz.localize(datetime.strptime("3:00_20150309", '%H:%M_%Y%m%d'))
        actual_time = parseATTime("midnight_tomorrow+3h", self.specified_tz)
        self.assertEqual(actual_time, expected_time)

    def test_should_return_current_time(self):
        expected_time = self.default_tz.localize(datetime.strptime("12:00_20150308", '%H:%M_%Y%m%d'))
        actual_time = parseATTime("now")
        self.assertEqual(actual_time, expected_time)

    def test_now_should_respect_tz(self):
        expected_time = self.specified_tz.localize(datetime.strptime("12:00_20150308", '%H:%M_%Y%m%d'))
        actual_time = parseATTime("now", self.specified_tz)
        self.assertEqual(actual_time, expected_time)

    def test_should_handle_dst_boundary(self):
        expected_time = self.specified_tz.localize(datetime.strptime("03:00_20150308", '%H:%M_%Y%m%d'))
        actual_time = parseATTime("midnight+2h", self.specified_tz)
        self.assertEqual(actual_time, expected_time)

MOCK_DATE = datetime(2015, 1, 1, 11, 00)

class AnotherMockedDateTime(datetime):
    def __new__(cls, *args, **kwargs):
        return datetime.__new__(datetime, *args, **kwargs)
    @classmethod
    def now(cls):
        return cls(2015, 1, 1, 11, 0, 0)

@mock.patch('graphite.render.attime.datetime', AnotherMockedDateTime)
class parseTimeReferenceTest(TestCase):

    def test_parse_empty_return_now(self):
        time_ref = parseTimeReference('')
        self.assertEquals(time_ref, MOCK_DATE)

    def test_parse_None_return_now(self):
        time_ref = parseTimeReference(None)
        self.assertEquals(time_ref, MOCK_DATE)

    def test_parse_random_string_raise_Exception(self):
        with self.assertRaises(Exception):
            time_ref = parseTimeReference("random")

    def test_parse_now_return_now(self):
        time_ref = parseTimeReference("now")
        self.assertEquals(time_ref, MOCK_DATE)

    def test_parse_colon_raises_ValueError(self):
        with self.assertRaises(ValueError):
            time_ref = parseTimeReference(":")

    def test_parse_hour_return_hour_of_today(self):
        time_ref = parseTimeReference("8:50")
        expected = datetime(MOCK_DATE.year, MOCK_DATE.month, MOCK_DATE.day, 8, 50)
        self.assertEquals(time_ref, expected)

    def test_parse_hour_am(self):
        time_ref = parseTimeReference("8:50am")
        expected = datetime(MOCK_DATE.year, MOCK_DATE.month, MOCK_DATE.day, 8, 50)
        self.assertEquals(time_ref, expected)

    def test_parse_hour_pm(self):
        time_ref = parseTimeReference("8:50pm")
        expected = datetime(MOCK_DATE.year, MOCK_DATE.month, MOCK_DATE.day, 20, 50)
        self.assertEquals(time_ref, expected)

    def test_parse_noon(self):
        time_ref = parseTimeReference("noon")
        expected = datetime(MOCK_DATE.year, MOCK_DATE.month, MOCK_DATE.day, 12, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_midnight(self):
        time_ref = parseTimeReference("midnight")
        expected = datetime(MOCK_DATE.year, MOCK_DATE.month, MOCK_DATE.day, 0, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_teatime(self):
        time_ref = parseTimeReference("teatime")
        expected = datetime(MOCK_DATE.year, MOCK_DATE.month, MOCK_DATE.day, 16, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_yesterday(self):
        time_ref = parseTimeReference("yesterday")
        expected = datetime(2014, 12, 31, 0, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_tomorrow(self):
        time_ref = parseTimeReference("tomorrow")
        expected = datetime(2015, 1, 2, 0, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_MM_slash_DD_slash_YY(self):
        time_ref = parseTimeReference("02/25/15")
        expected = datetime(2015, 2, 25, 0, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_MM_slash_DD_slash_YYYY(self):
        time_ref = parseTimeReference("02/25/2015")
        expected = datetime(2015, 2, 25, 0, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_YYYYMMDD(self):
        time_ref = parseTimeReference("20140606")
        expected = datetime(2014, 6, 6, 0, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_MonthName_DayOfMonth_onedigits(self):
        time_ref = parseTimeReference("january8")
        expected = datetime(2015, 1, 8, 0, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_MonthName_DayOfMonth_twodigits(self):
        time_ref = parseTimeReference("january10")
        expected = datetime(2015, 1, 10, 0, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_MonthName_DayOfMonth_threedigits_raise_ValueError(self):
        with self.assertRaises(ValueError):
            time_ref = parseTimeReference("january800")

    def test_parse_MonthName_without_DayOfMonth_raise_Exception(self):
        with self.assertRaises(Exception):
            time_ref = parseTimeReference("january")

    def test_parse_monday_return_monday_before_now(self):
        time_ref = parseTimeReference("monday")
        expected = datetime(2014, 12, 29, 0, 0)
        self.assertEquals(time_ref, expected)

class parseTimeOffsetTest(TestCase):

    def test_parse_None_returns_empty_timedelta(self):
        time_ref = parseTimeOffset(None)
        expected = timedelta(0)
        self.assertEquals(time_ref, expected)

    def test_parse_integer_raises_TypeError(self):
        with self.assertRaises(TypeError):
            time_ref = parseTimeOffset(1)

    def test_parse_string_starting_neither_with_minus_nor_digit_raises_KeyError(self):
        with self.assertRaises(KeyError):
            time_ref = parseTimeOffset("Something")

    def test_parse_m_as_unit_raises_Exception(self):
        with self.assertRaises(Exception):
            time_ref = parseTimeOffset("1m")

    def test_parse_digits_only_raises_exception(self):
        with self.assertRaises(Exception):
            time_ref = parseTimeOffset("10")

    def test_parse_alpha_only_raises_KeyError(self):
        with self.assertRaises(KeyError):
            time_ref = parseTimeOffset("month")

    def test_parse_minus_only_returns_zero(self):
        time_ref = parseTimeOffset("-")
        expected = timedelta(0)
        self.assertEquals(time_ref, expected)

    def test_parse_plus_only_returns_zero(self):
        time_ref = parseTimeOffset("+")
        expected = timedelta(0)
        self.assertEquals(time_ref, expected)

    def test_parse_ten_days(self):
        time_ref = parseTimeOffset("10days")
        expected = timedelta(10)
        self.assertEquals(time_ref, expected)

    def test_parse_zero_days(self):
        time_ref = parseTimeOffset("0days")
        expected = timedelta(0)
        self.assertEquals(time_ref, expected)

    def test_parse_minus_ten_days(self):
        time_ref = parseTimeOffset("-10days")
        expected = timedelta(-10)
        self.assertEquals(time_ref, expected)

    def test_parse_five_seconds(self):
        time_ref = parseTimeOffset("5seconds")
        expected = timedelta(seconds=5)
        self.assertEquals(time_ref, expected)

    def test_parse_five_minutes(self):
        time_ref = parseTimeOffset("5minutes")
        expected = timedelta(minutes=5)
        self.assertEquals(time_ref, expected)

    def test_parse_five_hours(self):
        time_ref = parseTimeOffset("5hours")
        expected = timedelta(hours=5)
        self.assertEquals(time_ref, expected)

    def test_parse_five_weeks(self):
        time_ref = parseTimeOffset("5weeks")
        expected = timedelta(weeks=5)
        self.assertEquals(time_ref, expected)

    def test_parse_one_month_returns_thirty_days(self):
        time_ref = parseTimeOffset("1month")
        expected = timedelta(30)
        self.assertEquals(time_ref, expected)

    def test_parse_two_months_returns_sixty_days(self):
        time_ref = parseTimeOffset("2months")
        expected = timedelta(60)
        self.assertEquals(time_ref, expected)

    def test_parse_twelve_months_returns_360_days(self):
        time_ref = parseTimeOffset("12months")
        expected = timedelta(360)
        self.assertEquals(time_ref, expected)

    def test_parse_one_year_returns_365_days(self):
        time_ref = parseTimeOffset("1year")
        expected = timedelta(365)
        self.assertEquals(time_ref, expected)

    def test_parse_two_years_returns_730_days(self):
        time_ref = parseTimeOffset("2years")
        expected = timedelta(730)
        self.assertEquals(time_ref, expected)

class getUnitStringTest(TestCase):

    def test_get_seconds(self):
        test_cases = ['s', 'se', 'sec', 'second', 'seconds']
        for test_case in test_cases:
            result = getUnitString(test_case)
            self.assertEquals(result, 'seconds')

    def test_get_minutes(self):
        test_cases = ['min', 'minute', 'minutes']
        for test_case in test_cases:
            result = getUnitString(test_case)
            self.assertEquals(result, 'minutes')

    def test_get_hours(self):
        test_cases = ['h', 'ho', 'hour', 'hours']
        for test_case in test_cases:
            result = getUnitString(test_case)
            self.assertEquals(result, 'hours')

    def test_get_days(self):
        test_cases = ['d', 'da', 'day', 'days']
        for test_case in test_cases:
            result = getUnitString(test_case)
            self.assertEquals(result, 'days')

    def test_get_weeks(self):
        test_cases = ['w', 'we', 'week', 'weeks']
        for test_case in test_cases:
            result = getUnitString(test_case)
            self.assertEquals(result, 'weeks')

    def test_get_months(self):
        test_cases = ['mon', 'month', 'months']
        for test_case in test_cases:
            result = getUnitString(test_case)
            self.assertEquals(result, 'months')

    def test_get_years(self):
        test_cases = ['y', 'ye', 'year', 'years']
        for test_case in test_cases:
            result = getUnitString(test_case)
            self.assertEquals(result, 'years')

    def test_m_raises_Exception(self):
        with self.assertRaises(Exception):
            result = getUnitString("m")

    def test_integer_raises_Exception(self):
        with self.assertRaises(Exception):
            result = getUnitString(1)

class parseATTimeTest(TestCase):

    @unittest.expectedFailure
    def test_parse_noon_plus_yesterday(self):
        time_ref = parseATTime("noon+yesterday")
        expected = datetime(MOCK_DATE.year, MOCK_DATE.month, MOCK_DATE.day - 1, 12, 00)
        self.assertEquals(time_ref, expected)
<commit_msg>fixing lint violation<commit_after>try:
    import unittest2 as unittest
except ImportError:
    import unittest
from graphite.render.attime import parseTimeReference, parseATTime, parseTimeOffset, getUnitString

from datetime import datetime, timedelta

from django.utils import timezone
from django.test import TestCase
import pytz
import mock

class MockedDateTime(datetime):
    def __new__(cls, *args, **kwargs):
        return datetime.__new__(datetime, *args, **kwargs)
    @classmethod
    def now(cls):
        return cls(2015, 3, 8, 12, 0, 0)

@mock.patch('graphite.render.attime.datetime', MockedDateTime)
class ATTimeTimezoneTests(TestCase):
    default_tz = timezone.get_current_timezone()
    specified_tz = pytz.timezone("America/Los_Angeles")
    def test_should_return_absolute_time(self):
        time_string = '12:0020150308'

        expected_time = self.default_tz.localize(datetime.strptime(time_string,'%H:%M%Y%m%d'))
        actual_time = parseATTime(time_string)
        self.assertEqual(actual_time, expected_time)

    def test_absolute_time_should_respect_tz(self):
        time_string = '12:0020150308'
        expected_time = self.specified_tz.localize(datetime.strptime(time_string, '%H:%M%Y%m%d'))
        actual_time = parseATTime(time_string, self.specified_tz)
        self.assertEqual(actual_time, expected_time)

    def test_midnight(self):
        expected_time = self.specified_tz.localize(datetime.strptime("0:00_20150308", '%H:%M_%Y%m%d'))
        actual_time = parseATTime("midnight", self.specified_tz)
        self.assertEqual(actual_time, expected_time)

    def test_offset_with_tz(self):
        expected_time = self.specified_tz.localize(datetime.strptime("5:00_20150308", '%H:%M_%Y%m%d'))
        actual_time = parseATTime("midnight+5h", self.specified_tz)
        self.assertEqual(actual_time, expected_time)

    def test_relative_day_with_tz(self):
        expected_time = self.specified_tz.localize(datetime.strptime("0:00_20150309", '%H:%M_%Y%m%d'))
        actual_time = parseATTime("midnight_tomorrow", self.specified_tz)
        self.assertEqual(actual_time, expected_time)

    def test_relative_day_and_offset_with_tz(self):
        expected_time = self.specified_tz.localize(datetime.strptime("3:00_20150309", '%H:%M_%Y%m%d'))
        actual_time = parseATTime("midnight_tomorrow+3h", self.specified_tz)
        self.assertEqual(actual_time, expected_time)

    def test_should_return_current_time(self):
        expected_time = self.default_tz.localize(datetime.strptime("12:00_20150308", '%H:%M_%Y%m%d'))
        actual_time = parseATTime("now")
        self.assertEqual(actual_time, expected_time)

    def test_now_should_respect_tz(self):
        expected_time = self.specified_tz.localize(datetime.strptime("12:00_20150308", '%H:%M_%Y%m%d'))
        actual_time = parseATTime("now", self.specified_tz)
        self.assertEqual(actual_time, expected_time)

    def test_should_handle_dst_boundary(self):
        expected_time = self.specified_tz.localize(datetime.strptime("03:00_20150308", '%H:%M_%Y%m%d'))
        actual_time = parseATTime("midnight+2h", self.specified_tz)
        self.assertEqual(actual_time, expected_time)

MOCK_DATE = datetime(2015, 1, 1, 11, 00)

class AnotherMockedDateTime(datetime):
    def __new__(cls, *args, **kwargs):
        return datetime.__new__(datetime, *args, **kwargs)
    @classmethod
    def now(cls):
        return cls(2015, 1, 1, 11, 0, 0)

@mock.patch('graphite.render.attime.datetime', AnotherMockedDateTime)
class parseTimeReferenceTest(TestCase):

    def test_parse_empty_return_now(self):
        time_ref = parseTimeReference('')
        self.assertEquals(time_ref, MOCK_DATE)

    def test_parse_None_return_now(self):
        time_ref = parseTimeReference(None)
        self.assertEquals(time_ref, MOCK_DATE)

    def test_parse_random_string_raise_Exception(self):
        with self.assertRaises(Exception):
            time_ref = parseTimeReference("random")

    def test_parse_now_return_now(self):
        time_ref = parseTimeReference("now")
        self.assertEquals(time_ref, MOCK_DATE)

    def test_parse_colon_raises_ValueError(self):
        with self.assertRaises(ValueError):
            time_ref = parseTimeReference(":")

    def test_parse_hour_return_hour_of_today(self):
        time_ref = parseTimeReference("8:50")
        expected = datetime(MOCK_DATE.year, MOCK_DATE.month, MOCK_DATE.day, 8, 50)
        self.assertEquals(time_ref, expected)

    def test_parse_hour_am(self):
        time_ref = parseTimeReference("8:50am")
        expected = datetime(MOCK_DATE.year, MOCK_DATE.month, MOCK_DATE.day, 8, 50)
        self.assertEquals(time_ref, expected)

    def test_parse_hour_pm(self):
        time_ref = parseTimeReference("8:50pm")
        expected = datetime(MOCK_DATE.year, MOCK_DATE.month, MOCK_DATE.day, 20, 50)
        self.assertEquals(time_ref, expected)

    def test_parse_noon(self):
        time_ref = parseTimeReference("noon")
        expected = datetime(MOCK_DATE.year, MOCK_DATE.month, MOCK_DATE.day, 12, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_midnight(self):
        time_ref = parseTimeReference("midnight")
        expected = datetime(MOCK_DATE.year, MOCK_DATE.month, MOCK_DATE.day, 0, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_teatime(self):
        time_ref = parseTimeReference("teatime")
        expected = datetime(MOCK_DATE.year, MOCK_DATE.month, MOCK_DATE.day, 16, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_yesterday(self):
        time_ref = parseTimeReference("yesterday")
        expected = datetime(2014, 12, 31, 0, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_tomorrow(self):
        time_ref = parseTimeReference("tomorrow")
        expected = datetime(2015, 1, 2, 0, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_MM_slash_DD_slash_YY(self):
        time_ref = parseTimeReference("02/25/15")
        expected = datetime(2015, 2, 25, 0, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_MM_slash_DD_slash_YYYY(self):
        time_ref = parseTimeReference("02/25/2015")
        expected = datetime(2015, 2, 25, 0, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_YYYYMMDD(self):
        time_ref = parseTimeReference("20140606")
        expected = datetime(2014, 6, 6, 0, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_MonthName_DayOfMonth_onedigits(self):
        time_ref = parseTimeReference("january8")
        expected = datetime(2015, 1, 8, 0, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_MonthName_DayOfMonth_twodigits(self):
        time_ref = parseTimeReference("january10")
        expected = datetime(2015, 1, 10, 0, 0)
        self.assertEquals(time_ref, expected)

    def test_parse_MonthName_DayOfMonth_threedigits_raise_ValueError(self):
        with self.assertRaises(ValueError):
            time_ref = parseTimeReference("january800")

    def test_parse_MonthName_without_DayOfMonth_raise_Exception(self):
        with self.assertRaises(Exception):
            time_ref = parseTimeReference("january")

    def test_parse_monday_return_monday_before_now(self):
        time_ref = parseTimeReference("monday")
        expected = datetime(2014, 12, 29, 0, 0)
        self.assertEquals(time_ref, expected)

class parseTimeOffsetTest(TestCase):

    def test_parse_None_returns_empty_timedelta(self):
        time_ref = parseTimeOffset(None)
        expected = timedelta(0)
        self.assertEquals(time_ref, expected)

    def test_parse_integer_raises_TypeError(self):
        with self.assertRaises(TypeError):
            time_ref = parseTimeOffset(1)

    def test_parse_string_starting_neither_with_minus_nor_digit_raises_KeyError(self):
        with self.assertRaises(KeyError):
            time_ref = parseTimeOffset("Something")

    def test_parse_m_as_unit_raises_Exception(self):
        with self.assertRaises(Exception):
            time_ref = parseTimeOffset("1m")

    def test_parse_digits_only_raises_exception(self):
        with self.assertRaises(Exception):
            time_ref = parseTimeOffset("10")

    def test_parse_alpha_only_raises_KeyError(self):
        with self.assertRaises(KeyError):
            time_ref = parseTimeOffset("month")

    def test_parse_minus_only_returns_zero(self):
        time_ref = parseTimeOffset("-")
        expected = timedelta(0)
        self.assertEquals(time_ref, expected)

    def test_parse_plus_only_returns_zero(self):
        time_ref = parseTimeOffset("+")
        expected = timedelta(0)
        self.assertEquals(time_ref, expected)

    def test_parse_ten_days(self):
        time_ref = parseTimeOffset("10days")
        expected = timedelta(10)
        self.assertEquals(time_ref, expected)

    def test_parse_zero_days(self):
        time_ref = parseTimeOffset("0days")
        expected = timedelta(0)
        self.assertEquals(time_ref, expected)

    def test_parse_minus_ten_days(self):
        time_ref = parseTimeOffset("-10days")
        expected = timedelta(-10)
        self.assertEquals(time_ref, expected)

    def test_parse_five_seconds(self):
        time_ref = parseTimeOffset("5seconds")
        expected = timedelta(seconds=5)
        self.assertEquals(time_ref, expected)

    def test_parse_five_minutes(self):
        time_ref = parseTimeOffset("5minutes")
        expected = timedelta(minutes=5)
        self.assertEquals(time_ref, expected)

    def test_parse_five_hours(self):
        time_ref = parseTimeOffset("5hours")
        expected = timedelta(hours=5)
        self.assertEquals(time_ref, expected)

    def test_parse_five_weeks(self):
        time_ref = parseTimeOffset("5weeks")
        expected = timedelta(weeks=5)
        self.assertEquals(time_ref, expected)

    def test_parse_one_month_returns_thirty_days(self):
        time_ref = parseTimeOffset("1month")
        expected = timedelta(30)
        self.assertEquals(time_ref, expected)

    def test_parse_two_months_returns_sixty_days(self):
        time_ref = parseTimeOffset("2months")
        expected = timedelta(60)
        self.assertEquals(time_ref, expected)

    def test_parse_twelve_months_returns_360_days(self):
        time_ref = parseTimeOffset("12months")
        expected = timedelta(360)
        self.assertEquals(time_ref, expected)

    def test_parse_one_year_returns_365_days(self):
        time_ref = parseTimeOffset("1year")
        expected = timedelta(365)
        self.assertEquals(time_ref, expected)

    def test_parse_two_years_returns_730_days(self):
        time_ref = parseTimeOffset("2years")
        expected = timedelta(730)
        self.assertEquals(time_ref, expected)

class getUnitStringTest(TestCase):

    def test_get_seconds(self):
        test_cases = ['s', 'se', 'sec', 'second', 'seconds']
        for test_case in test_cases:
            result = getUnitString(test_case)
            self.assertEquals(result, 'seconds')

    def test_get_minutes(self):
        test_cases = ['min', 'minute', 'minutes']
        for test_case in test_cases:
            result = getUnitString(test_case)
            self.assertEquals(result, 'minutes')

    def test_get_hours(self):
        test_cases = ['h', 'ho', 'hour', 'hours']
        for test_case in test_cases:
            result = getUnitString(test_case)
            self.assertEquals(result, 'hours')

    def test_get_days(self):
        test_cases = ['d', 'da', 'day', 'days']
        for test_case in test_cases:
            result = getUnitString(test_case)
            self.assertEquals(result, 'days')

    def test_get_weeks(self):
        test_cases = ['w', 'we', 'week', 'weeks']
        for test_case in test_cases:
            result = getUnitString(test_case)
            self.assertEquals(result, 'weeks')

    def test_get_months(self):
        test_cases = ['mon', 'month', 'months']
        for test_case in test_cases:
            result = getUnitString(test_case)
            self.assertEquals(result, 'months')

    def test_get_years(self):
        test_cases = ['y', 'ye', 'year', 'years']
        for test_case in test_cases:
            result = getUnitString(test_case)
            self.assertEquals(result, 'years')

    def test_m_raises_Exception(self):
        with self.assertRaises(Exception):
            result = getUnitString("m")

    def test_integer_raises_Exception(self):
        with self.assertRaises(Exception):
            result = getUnitString(1)

class parseATTimeTest(TestCase):

    @unittest.expectedFailure
    def test_parse_noon_plus_yesterday(self):
        time_ref = parseATTime("noon+yesterday")
        expected = datetime(MOCK_DATE.year, MOCK_DATE.month, MOCK_DATE.day - 1, 12, 00)
        self.assertEquals(time_ref, expected)
<|endoftext|>