<commit_before>import datetime
from dateutil.parser import parse
from dateutil.tz import tzutc

from marquise import Marquise

import ceilometer_publisher_vaultaire.siphash
import ceilometer_publisher_vaultaire.consolidated as consolidated

KEYS_TO_DELETE = [
    "timestamp",
    "volume",
    "created_at",
    "updated_at",
    "id",
    "size",
]

# FIXME: refactor the functions in this module in such a way that they
# can log.

def process_sample(sample):
    processed = []
    consolidated_sample = None
    try:
        if "event_type" in sample["resource_metadata"]:
            consolidated_sample = process_consolidated_event(sample)
        else:
            consolidated_sample = process_consolidated_pollster(sample)
    except Exception:
        # log here
        pass
    if consolidated_sample is not None:
        processed.append(consolidated_sample)
    processed.append(process_raw(sample))
    return processed

def remove_extraneous(sourcedict):
    for k in KEYS_TO_DELETE:
        sourcedict.pop(k, None)

def process_raw(sample):
    """
    Process a sample into a raw (not consolidated) datapoint. Makes
    destructive updates to sample.
    """
    event_type = sample["resource_metadata"].get("event_type", None)

    flavor_type = get_flavor_type(sample)

    id_elements = [
        sample["resource_id"],
        sample["project_id"],
        sample["name"],
        sample["unit"],
        event_type,
        flavor_type,
    ]

    # Filter out Nones and stringify everything so we don't get TypeErrors on concatenation
    id_elements = [ str(x) for x in id_elements if x is not None ]

    # Generate the unique identifer for the sample
    identifier = "".join(id_elements)
    address = Marquise.hash_identifier(identifier)

    # Sanitize timestamp (will parse timestamp to nanoseconds since epoch)
    timestamp = sanitize_timestamp(sample["timestamp"])

    # Our payload is the volume (later parsed to
    # "counter_volume" in ceilometer)
    payload = sanitize(sample["volume"])

    # Rebuild the sample as a source dict
    sourcedict = dict(sample)

    # Vaultaire cares about the datatype of the payload
    if type(payload) == float:
        sourcedict["_float"] = 1

    # Cast unit as a special metadata type
    sourcedict["_unit"] = sanitize(sourcedict.pop("unit", None))

    # If it's a cumulative value, we need to tell vaultaire
    if sourcedict["type"] == "cumulative":
        sourcedict["_counter"] = 1

    # Cast Identifier sections with unique names, in case of
    # metadata overlap

    # XXX: why do we call these counter_* even when they're
    # not counters?
    sourcedict["counter_name"] = sanitize(sourcedict.pop("name"))
    sourcedict["counter_type"] = sanitize(sourcedict.pop("type"))

    remove_extraneous(sourcedict)

    # Remove the original resource_metadata and substitute
    # our own flattened version
    sourcedict.update(flatten(sourcedict.pop("resource_metadata")))
    sourcedict = flatten(sourcedict)
    for k, v in sourcedict.items():
        sourcedict[k] = sanitize(str(v))
    return (address, sourcedict, timestamp, payload)

def process_consolidated_pollster(sample):
    # Pull out and clean fields which are always present
    name         = sample["name"]
    project_id   = sample["project_id"]
    resource_id  = sample["resource_id"]
    metadata     = sample["resource_metadata"]
    ## Cast unit as a special metadata type
    counter_unit = sample["unit"]
    counter_type = sample["type"]
    ## Sanitize timestamp (will parse timestamp to nanoseconds since epoch)
    timestamp    = sanitize_timestamp(sample["timestamp"])

    # We use a siphash of the instance_type for instance pollsters
    if name.startswith("instance"):
        payload = canonical_siphash(metadata["instance_type"])
    elif name.startswith("volume.size"):
        payload = consolidated.volumeToRawPayload(sanitize(sample["volume"]))
    elif name.startswith("ip.floating"):
        payload = consolidated.RAW_PAYLOAD_IP_ALLOC
    else:
        payload = sanitize(sample["volume"])

    # Build the source dict
    sourcedict = {}
    sourcedict["_consolidated"] = 1
    sourcedict["project_id"]    = project_id
    sourcedict["resource_id"]   = resource_id
    sourcedict["counter_name"]  = name
    sourcedict["counter_unit"]  = counter_unit
    sourcedict["counter_type"]  = counter_type
    sourcedict["display_name"]  = metadata.get("display_name", "")
    # Vaultaire does not care about the datatype of the payload, so we
    # need to specify it in metadata.
    if type(payload) == float:
        sourcedict["_float"] = 1

    # If it's a cumulative value, this needs to go in the metadata as
    # well.
    if counter_type == "cumulative":
        sourcedict["_counter"] = 1

    for k, v in sourcedict.items():
        sourcedict[k] = sanitize(str(v))

    # Common elements to all messages are r_id + p_id + counter_(name,
    # type, unit)
    id_elements = [
        resource_id,
        project_id,
        name,
        counter_type,
        counter_unit,
        "_consolidated",
    ]

    # Filter out Nones and stringify everything so we don't get
    # TypeErrors on concatenation
    id_elements = [ str(x) for x in id_elements if x is not None ]

    # Generate the unique identifer for the sample
    identifier = "".join(id_elements)
    address = Marquise.hash_identifier(identifier)
    return (address, sourcedict, timestamp, payload)

def process_consolidated_event(sample):
    # Pull out and clean fields which are always present
    name         = sample["name"]
    project_id   = sample["project_id"]
    resource_id  = sample["resource_id"]
    metadata     = sample["resource_metadata"]
    ## Cast unit as a special metadata type
    counter_unit = sample["unit"]
    counter_type = sample["type"]
    ## Sanitize timestamp (will parse timestamp to nanoseconds since epoch)
    timestamp    = sanitize_timestamp(sample["timestamp"])

    # We use the instance_type_id for instance events
    if name.startswith("instance"):
        payload = consolidated.constructPayload(metadata["event_type"], metadata.get("message",""), metadata["instance_type_id"])
    elif name.startswith("volume.size"):
        payload = consolidated.constructPayload(metadata["event_type"], metadata.get("status",""), consolidated.volumeToRawPayload(sample["volume"]))
    elif name.startswith("ip.floating"):
        payload = consolidated.constructPayload(metadata["event_type"], "", 1)
    else:
        payload = sanitize(sample["volume"])

    # Build the source dict
    sourcedict = {}
    sourcedict["_event"]        = 1
    sourcedict["_consolidated"] = 1
    sourcedict["project_id"]    = project_id
    sourcedict["resource_id"]   = resource_id
    sourcedict["counter_name"]  = name
    sourcedict["counter_unit"]  = counter_unit
    sourcedict["counter_type"]  = counter_type
    sourcedict["display_name"]  = metadata.get("display_name", "")
    ## Vaultaire cares about the datatype of the payload
    if type(payload) == float:
        sourcedict["_float"] = 1

    ## If it's a cumulative value, we need to tell vaultaire
    if counter_type == "cumulative":
        sourcedict["_counter"] = 1

    for k, v in sourcedict.items():
        sourcedict[k] = sanitize(str(v))

    # Common elements to all messages are r_id + p_id + counter_(name,
    # type, unit)
    id_elements = [
        resource_id,
        project_id,
        name,
        counter_type,
        counter_unit,
        "_event",
        "_consolidated",
    ]

    # Filter out Nones and stringify everything so we don't get
    # TypeErrors on concatenation
    id_elements = [ str(x) for x in id_elements if x is not None ]

    # Generate the unique identifer for the sample
    identifier = "".join(id_elements)
    address = Marquise.hash_identifier(identifier)
    return (address, sourcedict, timestamp, payload)

def sanitize(v):
    """Sanitize a value into something that Marquise can use. This weeds
    out None keys/values, and ensures that timestamps are consistently
    formatted.
    """
    if v is None:
        return ""
    if type(v) is bool:
        if v:
            return 1
        else:
            return 0
    if type(v) is unicode:
        v = str(v)
    if type(v) is str: # XXX: will be incorrect/ambiguous in Python 3.
        v = v.replace(":","-")
        v = v.replace(",","-")
    return v

def flatten(n, prefix=""):
    """Take a (potentially) nested dictionary and flatten it into a single
    level. Also remove any keys/values that Marquise/Vaultaire can't handle.
    """
    flattened_dict = {}
    for k,v in n.items():
        k = sanitize(k)

        # Vaultaire doesn't care about generated URLs for Ceilometer
        # API references.
        if str(k) == "links":
            continue
        if str(k).endswith('_url'):
            continue

        # Vaultaire doesn't want values if they have no contents.
        if v is None:
            continue

        # If key has a parent, concatenate it into the new keyname.
        if prefix != "":
            k = "{}-{}".format(prefix,k)

        # This was previously a check for __iter__, but strings have those now,
        # so let's just check for dict-ness instead. No good on lists anyway.
        if type(v) is not dict:
            v = sanitize(v)
            flattened_dict[k] = v
        else:
            flattened_dict.update(flatten(v, k))
    return flattened_dict

def sanitize_timestamp(v):
    """Convert a timestamp value into a standard form. Does no exception
    handling, as we really want to know if this fails."""
    if type(v) is unicode:
        v = str(v)
    # Try and take a value and use dateutil to parse it.  If there's no TZ
    # spec in the string, assume it's UTC because that's what Ceilometer
    # uses.
    # Eg. 2014-08-10T12:14:13Z      # timezone-aware
    # Eg. 2014-08-10 12:14:13       # timezone-naive
    # Eg. 2014-08-10 12:14:13+1000  # timezone-aware
    NANOSECONDS_PER_SECOND = 10**9
    if type(v) is datetime.datetime:
        timestamp = v
    else: # We have a string.
        timestamp = parse(v)
        if timestamp.tzinfo is None:
            timestamp = timestamp.replace(tzinfo=tzutc())
    # If we get here, we've successfully grabbed a datetime.
    # FIXME: use strftime
    epoch = datetime.datetime(1970, 1, 1, tzinfo=tzutc())
    time_since_epoch = (timestamp - epoch).total_seconds() # total_seconds() is in Py2.7 and later.
    return int(time_since_epoch * NANOSECONDS_PER_SECOND)

def canonical_siphash(the_thing):
    return ceilometer_publisher_vaultaire.siphash.SipHash24("0000000000000000", the_thing).hash()

def get_flavor_type(sample):
    flavor_type = None
    if "flavor" in sample:
        flavor_type = sample["flavor"].get("name", None)
    elif "instance_type" in sample:
        flavor_type = sample["instance_type"]
    return flavor_type
<commit_msg>Pollster sourcedict fix for borel<commit_after>import datetime
from dateutil.parser import parse
from dateutil.tz import tzutc

from marquise import Marquise

import ceilometer_publisher_vaultaire.siphash
import ceilometer_publisher_vaultaire.consolidated as consolidated

KEYS_TO_DELETE = [
    "timestamp",
    "volume",
    "created_at",
    "updated_at",
    "id",
    "size",
]

# FIXME: refactor the functions in this module in such a way that they
# can log.

def process_sample(sample):
    processed = []
    consolidated_sample = None
    try:
        if "event_type" in sample["resource_metadata"]:
            consolidated_sample = process_consolidated_event(sample)
        else:
            consolidated_sample = process_consolidated_pollster(sample)
    except Exception:
        # log here
        pass
    if consolidated_sample is not None:
        processed.append(consolidated_sample)
    processed.append(process_raw(sample))
    return processed

def remove_extraneous(sourcedict):
    for k in KEYS_TO_DELETE:
        sourcedict.pop(k, None)

def process_raw(sample):
    """
    Process a sample into a raw (not consolidated) datapoint. Makes
    destructive updates to sample.
    """
    event_type = sample["resource_metadata"].get("event_type", None)

    flavor_type = get_flavor_type(sample)

    id_elements = [
        sample["resource_id"],
        sample["project_id"],
        sample["name"],
        sample["unit"],
        event_type,
        flavor_type,
    ]

    # Filter out Nones and stringify everything so we don't get TypeErrors on concatenation
    id_elements = [ str(x) for x in id_elements if x is not None ]

    # Generate the unique identifer for the sample
    identifier = "".join(id_elements)
    address = Marquise.hash_identifier(identifier)

    # Sanitize timestamp (will parse timestamp to nanoseconds since epoch)
    timestamp = sanitize_timestamp(sample["timestamp"])

    # Our payload is the volume (later parsed to
    # "counter_volume" in ceilometer)
    payload = sanitize(sample["volume"])

    # Rebuild the sample as a source dict
    sourcedict = dict(sample)

    # Vaultaire cares about the datatype of the payload
    if type(payload) == float:
        sourcedict["_float"] = 1

    # Cast unit as a special metadata type
    sourcedict["_unit"] = sanitize(sourcedict.pop("unit", None))

    # If it's a cumulative value, we need to tell vaultaire
    if sourcedict["type"] == "cumulative":
        sourcedict["_counter"] = 1

    # Cast Identifier sections with unique names, in case of
    # metadata overlap

    # XXX: why do we call these counter_* even when they're
    # not counters?
    sourcedict["counter_name"] = sanitize(sourcedict.pop("name"))
    sourcedict["counter_type"] = sanitize(sourcedict.pop("type"))

    remove_extraneous(sourcedict)

    # Remove the original resource_metadata and substitute
    # our own flattened version
    sourcedict.update(flatten(sourcedict.pop("resource_metadata")))
    sourcedict = flatten(sourcedict)
    for k, v in sourcedict.items():
        sourcedict[k] = sanitize(str(v))
    return (address, sourcedict, timestamp, payload)

def process_consolidated_pollster(sample):
    # Pull out and clean fields which are always present
    name         = sample["name"]
    project_id   = sample["project_id"]
    resource_id  = sample["resource_id"]
    metadata     = sample["resource_metadata"]
    ## Cast unit as a special metadata type
    counter_unit = sample["unit"]
    counter_type = sample["type"]
    ## Sanitize timestamp (will parse timestamp to nanoseconds since epoch)
    timestamp    = sanitize_timestamp(sample["timestamp"])

    # We use a siphash of the instance_type for instance pollsters
    if name.startswith("instance"):
        payload = canonical_siphash(metadata["instance_type"])
    elif name.startswith("volume.size"):
        payload = consolidated.volumeToRawPayload(sanitize(sample["volume"]))
    elif name.startswith("ip.floating"):
        payload = consolidated.RAW_PAYLOAD_IP_ALLOC
    else:
        payload = sanitize(sample["volume"])

    # Build the source dict
    sourcedict = {}
    sourcedict["_event"]        = 0
    sourcedict["_consolidated"] = 1
    sourcedict["project_id"]    = project_id
    sourcedict["resource_id"]   = resource_id
    sourcedict["counter_name"]  = name
    sourcedict["counter_unit"]  = counter_unit
    sourcedict["counter_type"]  = counter_type
    sourcedict["display_name"]  = metadata.get("display_name", "")
    # Vaultaire does not care about the datatype of the payload, so we
    # need to specify it in metadata.
    if type(payload) == float:
        sourcedict["_float"] = 1

    # If it's a cumulative value, this needs to go in the metadata as
    # well.
    if counter_type == "cumulative":
        sourcedict["_counter"] = 1

    for k, v in sourcedict.items():
        sourcedict[k] = sanitize(str(v))

    # Common elements to all messages are r_id + p_id + counter_(name,
    # type, unit)
    id_elements = [
        resource_id,
        project_id,
        name,
        counter_type,
        counter_unit,
        "_consolidated",
    ]

    # Filter out Nones and stringify everything so we don't get
    # TypeErrors on concatenation
    id_elements = [ str(x) for x in id_elements if x is not None ]

    # Generate the unique identifer for the sample
    identifier = "".join(id_elements)
    address = Marquise.hash_identifier(identifier)
    return (address, sourcedict, timestamp, payload)

def process_consolidated_event(sample):
    # Pull out and clean fields which are always present
    name         = sample["name"]
    project_id   = sample["project_id"]
    resource_id  = sample["resource_id"]
    metadata     = sample["resource_metadata"]
    ## Cast unit as a special metadata type
    counter_unit = sample["unit"]
    counter_type = sample["type"]
    ## Sanitize timestamp (will parse timestamp to nanoseconds since epoch)
    timestamp    = sanitize_timestamp(sample["timestamp"])

    # We use the instance_type_id for instance events
    if name.startswith("instance"):
        payload = consolidated.constructPayload(metadata["event_type"], metadata.get("message",""), metadata["instance_type_id"])
    elif name.startswith("volume.size"):
        payload = consolidated.constructPayload(metadata["event_type"], metadata.get("status",""), consolidated.volumeToRawPayload(sample["volume"]))
    elif name.startswith("ip.floating"):
        payload = consolidated.constructPayload(metadata["event_type"], "", 1)
    else:
        payload = sanitize(sample["volume"])

    # Build the source dict
    sourcedict = {}
    sourcedict["_event"]        = 1
    sourcedict["_consolidated"] = 1
    sourcedict["project_id"]    = project_id
    sourcedict["resource_id"]   = resource_id
    sourcedict["counter_name"]  = name
    sourcedict["counter_unit"]  = counter_unit
    sourcedict["counter_type"]  = counter_type
    sourcedict["display_name"]  = metadata.get("display_name", "")
    ## Vaultaire cares about the datatype of the payload
    if type(payload) == float:
        sourcedict["_float"] = 1

    ## If it's a cumulative value, we need to tell vaultaire
    if counter_type == "cumulative":
        sourcedict["_counter"] = 1

    for k, v in sourcedict.items():
        sourcedict[k] = sanitize(str(v))

    # Common elements to all messages are r_id + p_id + counter_(name,
    # type, unit)
    id_elements = [
        resource_id,
        project_id,
        name,
        counter_type,
        counter_unit,
        "_event",
        "_consolidated",
    ]

    # Filter out Nones and stringify everything so we don't get
    # TypeErrors on concatenation
    id_elements = [ str(x) for x in id_elements if x is not None ]

    # Generate the unique identifer for the sample
    identifier = "".join(id_elements)
    address = Marquise.hash_identifier(identifier)
    return (address, sourcedict, timestamp, payload)

def sanitize(v):
    """Sanitize a value into something that Marquise can use. This weeds
    out None keys/values, and ensures that timestamps are consistently
    formatted.
    """
    if v is None:
        return ""
    if type(v) is bool:
        if v:
            return 1
        else:
            return 0
    if type(v) is unicode:
        v = str(v)
    if type(v) is str: # XXX: will be incorrect/ambiguous in Python 3.
        v = v.replace(":","-")
        v = v.replace(",","-")
    return v

def flatten(n, prefix=""):
    """Take a (potentially) nested dictionary and flatten it into a single
    level. Also remove any keys/values that Marquise/Vaultaire can't handle.
    """
    flattened_dict = {}
    for k,v in n.items():
        k = sanitize(k)

        # Vaultaire doesn't care about generated URLs for Ceilometer
        # API references.
        if str(k) == "links":
            continue
        if str(k).endswith('_url'):
            continue

        # Vaultaire doesn't want values if they have no contents.
        if v is None:
            continue

        # If key has a parent, concatenate it into the new keyname.
        if prefix != "":
            k = "{}-{}".format(prefix,k)

        # This was previously a check for __iter__, but strings have those now,
        # so let's just check for dict-ness instead. No good on lists anyway.
        if type(v) is not dict:
            v = sanitize(v)
            flattened_dict[k] = v
        else:
            flattened_dict.update(flatten(v, k))
    return flattened_dict

def sanitize_timestamp(v):
    """Convert a timestamp value into a standard form. Does no exception
    handling, as we really want to know if this fails."""
    if type(v) is unicode:
        v = str(v)
    # Try and take a value and use dateutil to parse it.  If there's no TZ
    # spec in the string, assume it's UTC because that's what Ceilometer
    # uses.
    # Eg. 2014-08-10T12:14:13Z      # timezone-aware
    # Eg. 2014-08-10 12:14:13       # timezone-naive
    # Eg. 2014-08-10 12:14:13+1000  # timezone-aware
    NANOSECONDS_PER_SECOND = 10**9
    if type(v) is datetime.datetime:
        timestamp = v
    else: # We have a string.
        timestamp = parse(v)
        if timestamp.tzinfo is None:
            timestamp = timestamp.replace(tzinfo=tzutc())
    # If we get here, we've successfully grabbed a datetime.
    # FIXME: use strftime
    epoch = datetime.datetime(1970, 1, 1, tzinfo=tzutc())
    time_since_epoch = (timestamp - epoch).total_seconds() # total_seconds() is in Py2.7 and later.
    return int(time_since_epoch * NANOSECONDS_PER_SECOND)

def canonical_siphash(the_thing):
    return ceilometer_publisher_vaultaire.siphash.SipHash24("0000000000000000", the_thing).hash()

def get_flavor_type(sample):
    flavor_type = None
    if "flavor" in sample:
        flavor_type = sample["flavor"].get("name", None)
    elif "instance_type" in sample:
        flavor_type = sample["instance_type"]
    return flavor_type
<|endoftext|>


<commit_before>import datetime
import sys
import threading

from django.conf import settings
from django.core.management.commands.runserver import \
    Command as RunserverCommand
from django.utils import six
from django.utils.encoding import get_system_encoding

from channels import DEFAULT_CHANNEL_LAYER, channel_layers
from channels.handler import ViewConsumer
from channels.log import setup_logger
from channels.staticfiles import StaticFilesConsumer
from channels.worker import Worker

class Command(RunserverCommand):

    def add_arguments(self, parser):
        super(Command, self).add_arguments(parser)
        parser.add_argument('--noworker', action='store_false', dest='run_worker', default=True,
            help='Tells Django not to run a worker thread; you\'ll need to run one separately.')
        parser.add_argument('--noasgi', action='store_false', dest='use_asgi', default=True,
            help='Run the old WSGI-based runserver rather than the ASGI-based one')

    def handle(self, *args, **options):
        self.verbosity = options.get("verbosity", 1)
        self.logger = setup_logger('django.channels', self.verbosity)
        super(Command, self).handle(*args, **options)

    def inner_run(self, *args, **options):
        # Maybe they want the wsgi one?
        if not options.get("use_asgi", True):
            return RunserverCommand.inner_run(self, *args, **options)
        # Check a handler is registered for http reqs; if not, add default one
        self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER]
        self.channel_layer.registry.check_default(
            http_consumer=self.get_consumer(),
        )
        # Run checks
        self.stdout.write("Performing system checks...\n\n")
        self.check(display_num_errors=True)
        self.check_migrations()
        # Print helpful text
        quit_command = 'CTRL-BREAK' if sys.platform == 'win32' else 'CONTROL-C'
        now = datetime.datetime.now().strftime('%B %d, %Y - %X')
        if six.PY2:
            now = now.decode(get_system_encoding())
        self.stdout.write(now)
        self.stdout.write((
            "Django version %(version)s, using settings %(settings)r\n"
            "Starting Channels development server at http://%(addr)s:%(port)s/\n"
            "Channel layer %(layer)s\n"
            "Quit the server with %(quit_command)s.\n"
        ) % {
            "version": self.get_version(),
            "settings": settings.SETTINGS_MODULE,
            "addr": '[%s]' % self.addr if self._raw_ipv6 else self.addr,
            "port": self.port,
            "quit_command": quit_command,
            "layer": self.channel_layer,
        })

        # Launch workers as subthreads
        if options.get("run_worker", True):
            for _ in range(4):
                worker = WorkerThread(self.channel_layer, self.logger)
                worker.daemon = True
                worker.start()
        # Launch server in 'main' thread. Signals are disabled as it's still
        # actually a subthread under the autoreloader.
        self.logger.debug("Daphne running, listening on %s:%s", self.addr, self.port)
        try:
            from daphne.server import Server
            Server(
                channel_layer=self.channel_layer,
                host=self.addr,
                port=int(self.port),
                signal_handlers=not options['use_reloader'],
                action_logger=self.log_action,
                http_timeout=60,  # Shorter timeout than normal as it's dev
            ).run()
            self.logger.debug("Daphne exited")
        except KeyboardInterrupt:
            shutdown_message = options.get('shutdown_message', '')
            if shutdown_message:
                self.stdout.write(shutdown_message)
            return

    def log_action(self, protocol, action, details):
        """
        Logs various different kinds of requests to the console.
        """
        # All start with timestamp
        msg = "[%s] " % datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")
        # HTTP requests
        if protocol == "http" and action == "complete":
            msg += "HTTP %(method)s %(path)s %(status)s [%(time_taken).2f, %(client)s]\n" % details
            # Utilize terminal colors, if available
            if 200 <= details['status'] < 300:
                # Put 2XX first, since it should be the common case
                msg = self.style.HTTP_SUCCESS(msg)
            elif 100 <= details['status'] < 200:
                msg = self.style.HTTP_INFO(msg)
            elif details['status'] == 304:
                msg = self.style.HTTP_NOT_MODIFIED(msg)
            elif 300 <= details['status'] < 400:
                msg = self.style.HTTP_REDIRECT(msg)
            elif details['status'] == 404:
                msg = self.style.HTTP_NOT_FOUND(msg)
            elif 400 <= details['status'] < 500:
                msg = self.style.HTTP_BAD_REQUEST(msg)
            else:
                # Any 5XX, or any other response
                msg = self.style.HTTP_SERVER_ERROR(msg)
        # Websocket requests
        elif protocol == "websocket" and action == "connected":
            msg += "WebSocket CONNECT %(path)s [%(client)s]\n" % details
        elif protocol == "websocket" and action == "disconnected":
            msg += "WebSocket DISCONNECT %(path)s [%(client)s]\n" % details

        sys.stderr.write(msg)

    def get_consumer(self, *args, **options):
        """
        Returns the static files serving handler wrapping the default handler,
        if static files should be served. Otherwise just returns the default
        handler.
        """
        use_static_handler = options.get('use_static_handler', True)
        insecure_serving = options.get('insecure_serving', False)
        if use_static_handler and (settings.DEBUG or insecure_serving):
            return StaticFilesConsumer()
        else:
            return ViewConsumer()

class WorkerThread(threading.Thread):
    """
    Class that runs a worker
    """

    def __init__(self, channel_layer, logger):
        super(WorkerThread, self).__init__()
        self.channel_layer = channel_layer
        self.logger = logger

    def run(self):
        self.logger.debug("Worker thread running")
        worker = Worker(channel_layer=self.channel_layer, signal_handlers=False)
        worker.run()
        self.logger.debug("Worker thread exited")
<commit_msg>Use WSGI runserver if no channel backend configured<commit_after>import datetime
import sys
import threading

from django.conf import settings
from django.core.management.commands.runserver import \
    Command as RunserverCommand
from django.utils import six
from django.utils.encoding import get_system_encoding

from channels import DEFAULT_CHANNEL_LAYER, channel_layers
from channels.handler import ViewConsumer
from channels.log import setup_logger
from channels.staticfiles import StaticFilesConsumer
from channels.worker import Worker

class Command(RunserverCommand):

    def add_arguments(self, parser):
        super(Command, self).add_arguments(parser)
        parser.add_argument('--noworker', action='store_false', dest='run_worker', default=True,
            help='Tells Django not to run a worker thread; you\'ll need to run one separately.')
        parser.add_argument('--noasgi', action='store_false', dest='use_asgi', default=True,
            help='Run the old WSGI-based runserver rather than the ASGI-based one')

    def handle(self, *args, **options):
        self.verbosity = options.get("verbosity", 1)
        self.logger = setup_logger('django.channels', self.verbosity)
        super(Command, self).handle(*args, **options)

    def inner_run(self, *args, **options):
        # Maybe they want the wsgi one?
        if not options.get("use_asgi", True) or DEFAULT_CHANNEL_LAYER not in channel_layers:
            return RunserverCommand.inner_run(self, *args, **options)
        # Check a handler is registered for http reqs; if not, add default one
        self.channel_layer = channel_layers[DEFAULT_CHANNEL_LAYER]
        self.channel_layer.registry.check_default(
            http_consumer=self.get_consumer(),
        )
        # Run checks
        self.stdout.write("Performing system checks...\n\n")
        self.check(display_num_errors=True)
        self.check_migrations()
        # Print helpful text
        quit_command = 'CTRL-BREAK' if sys.platform == 'win32' else 'CONTROL-C'
        now = datetime.datetime.now().strftime('%B %d, %Y - %X')
        if six.PY2:
            now = now.decode(get_system_encoding())
        self.stdout.write(now)
        self.stdout.write((
            "Django version %(version)s, using settings %(settings)r\n"
            "Starting Channels development server at http://%(addr)s:%(port)s/\n"
            "Channel layer %(layer)s\n"
            "Quit the server with %(quit_command)s.\n"
        ) % {
            "version": self.get_version(),
            "settings": settings.SETTINGS_MODULE,
            "addr": '[%s]' % self.addr if self._raw_ipv6 else self.addr,
            "port": self.port,
            "quit_command": quit_command,
            "layer": self.channel_layer,
        })

        # Launch workers as subthreads
        if options.get("run_worker", True):
            for _ in range(4):
                worker = WorkerThread(self.channel_layer, self.logger)
                worker.daemon = True
                worker.start()
        # Launch server in 'main' thread. Signals are disabled as it's still
        # actually a subthread under the autoreloader.
        self.logger.debug("Daphne running, listening on %s:%s", self.addr, self.port)
        try:
            from daphne.server import Server
            Server(
                channel_layer=self.channel_layer,
                host=self.addr,
                port=int(self.port),
                signal_handlers=not options['use_reloader'],
                action_logger=self.log_action,
                http_timeout=60,  # Shorter timeout than normal as it's dev
            ).run()
            self.logger.debug("Daphne exited")
        except KeyboardInterrupt:
            shutdown_message = options.get('shutdown_message', '')
            if shutdown_message:
                self.stdout.write(shutdown_message)
            return

    def log_action(self, protocol, action, details):
        """
        Logs various different kinds of requests to the console.
        """
        # All start with timestamp
        msg = "[%s] " % datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")
        # HTTP requests
        if protocol == "http" and action == "complete":
            msg += "HTTP %(method)s %(path)s %(status)s [%(time_taken).2f, %(client)s]\n" % details
            # Utilize terminal colors, if available
            if 200 <= details['status'] < 300:
                # Put 2XX first, since it should be the common case
                msg = self.style.HTTP_SUCCESS(msg)
            elif 100 <= details['status'] < 200:
                msg = self.style.HTTP_INFO(msg)
            elif details['status'] == 304:
                msg = self.style.HTTP_NOT_MODIFIED(msg)
            elif 300 <= details['status'] < 400:
                msg = self.style.HTTP_REDIRECT(msg)
            elif details['status'] == 404:
                msg = self.style.HTTP_NOT_FOUND(msg)
            elif 400 <= details['status'] < 500:
                msg = self.style.HTTP_BAD_REQUEST(msg)
            else:
                # Any 5XX, or any other response
                msg = self.style.HTTP_SERVER_ERROR(msg)
        # Websocket requests
        elif protocol == "websocket" and action == "connected":
            msg += "WebSocket CONNECT %(path)s [%(client)s]\n" % details
        elif protocol == "websocket" and action == "disconnected":
            msg += "WebSocket DISCONNECT %(path)s [%(client)s]\n" % details

        sys.stderr.write(msg)

    def get_consumer(self, *args, **options):
        """
        Returns the static files serving handler wrapping the default handler,
        if static files should be served. Otherwise just returns the default
        handler.
        """
        use_static_handler = options.get('use_static_handler', True)
        insecure_serving = options.get('insecure_serving', False)
        if use_static_handler and (settings.DEBUG or insecure_serving):
            return StaticFilesConsumer()
        else:
            return ViewConsumer()

class WorkerThread(threading.Thread):
    """
    Class that runs a worker
    """

    def __init__(self, channel_layer, logger):
        super(WorkerThread, self).__init__()
        self.channel_layer = channel_layer
        self.logger = logger

    def run(self):
        self.logger.debug("Worker thread running")
        worker = Worker(channel_layer=self.channel_layer, signal_handlers=False)
        worker.run()
        self.logger.debug("Worker thread exited")
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# Author: CEF PNM Team
# License: TBD
# Copyright (c) 2012

#from __future__ import print_function
"""

module __Permeability__: Hagen Poiseuille permeability
========================================================================

.. warning:: The classes of this module should be loaded through the 'Algorithms.__init__.py' file.

"""

import scipy as sp
from .__LinearSolver__ import LinearSolver

class StokesFlow(LinearSolver):
    r"""

    Hagen Poiseuille permeability - Class to run an algorithm to find permeability on constructed networks

                        It returns pressure gradient inside the network.

    Parameter
    ----------
    -loglevel : int
        Level of the logger (10=Debug, 20=INFO, 30=Warning, 40=Error, 50=Critical)

    """

    def __init__(self,loglevel=10,**kwargs):
        r"""
        Initializing the class
        """
        super(StokesFlow,self).__init__(**kwargs)
        self._logger.info("Create Hagen Poiseuille permeability Object")

    def _setup(self,**params):
        r"""
        This function executes the essential mathods before building matrices in Linear solution
        """

        self._fluid = params['active_fluid']
        # Building hydraulic conductance
        g = self._fluid.get_throat_data(prop='hydraulic_conductance')
        s = self._fluid.get_throat_data(prop='occupancy')
        self._conductance = g*s+g*(-s)/1e3

    def _do_inner_iteration_stage(self):

        p = self._do_one_inner_iteration()
        self._fluid.set_pore_data(prop='pressure',data = p)

    def calc_eff_permeability_cubic(self,face1=1,face2=6):
        r"""
        This function calculates effective permeability of a cubic network between face1 and face2.
        face1 and face2 represent types of these two faces.

        """
        face1_pores = self._net.pore_properties['numbering'][self._net.pore_properties['type']==face1]
        face2_pores = self._net.pore_properties['numbering'][self._net.pore_properties['type']==face2]
        p = self._fluid.pore_conditions['pressure']
        P1 = p[face1_pores]
        P2 = p[face2_pores]
        delta_P = sp.absolute(sp.average(P2)-sp.average(P1))
        mu =sp.average(self._fluid.pore_conditions['viscosity'])

        coordx = self._net.pore_properties['coords'][:,0]
        coordy = self._net.pore_properties['coords'][:,1]
        coordz = self._net.pore_properties['coords'][:,2]
        if face1==1 or face1==6:
            L = sp.absolute(sp.unique(coordz[face1_pores])[0]-sp.unique(coordz[face2_pores])[0])*1e-6
            lx = (max(coordx[face1_pores]) - min(coordx[face1_pores]))*1e-6
            ly = (max(coordy[face1_pores]) - min(coordy[face1_pores]))*1e-6
            A = lx*ly
        elif face1==2 or face1==5:
            L = sp.absolute(sp.unique(coordx[face1])[0]-sp.unique(coordx[face2])[0])*1e-6
            lz = (max(coordz[face1_pores]) - min(coordz[face1_pores]))*1e-6
            ly = (max(coordy[face1_pores]) - min(coordy[face1_pores]))*1e-6
            A = lz*ly
        elif face1==3 or face1==4:
            L = sp.absolute(sp.unique(coordy[face1_pores])[0]-sp.unique(coordy[face2_pores])[0])*1e-6
            lx = (max(coordx[face1_pores]) - min(coordx[face1_pores]))*1e-6
            lz = (max(coordz[face1_pores]) - min(coordz[face1_pores]))*1e-6
            A = lx*lz

        fn = self._net.find_neighbor_pores(face1_pores)
        fn = fn[self._net.pore_properties['type'][fn]<1]
        ft = self._net.find_connecting_throat(face1_pores,fn)
        Q = sp.sum(self._fluid.throat_conditions['hydraulic_conductance'][ft]*sp.absolute(P1-p[fn]))
        Keff = Q*mu*L/(A*delta_P)
        return Keff<commit_msg>Effective permeability method for the cubic networks is removed from StokesFlow file.<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# Author: CEF PNM Team
# License: TBD
# Copyright (c) 2012

#from __future__ import print_function
"""

module __Permeability__: Hagen Poiseuille permeability
========================================================================

.. warning:: The classes of this module should be loaded through the 'Algorithms.__init__.py' file.

"""

import scipy as sp
from .__LinearSolver__ import LinearSolver

class StokesFlow(LinearSolver):
    r"""

    Hagen Poiseuille permeability - Class to run an algorithm to find permeability on constructed networks

                        It returns pressure gradient inside the network.

    Parameter
    ----------
    -loglevel : int
        Level of the logger (10=Debug, 20=INFO, 30=Warning, 40=Error, 50=Critical)

    """

    def __init__(self,loglevel=10,**kwargs):
        r"""
        Initializing the class
        """
        super(StokesFlow,self).__init__(**kwargs)
        self._logger.info("Create Hagen Poiseuille permeability Object")

    def _setup(self,**params):
        r"""
        This function executes the essential mathods before building matrices in Linear solution
        """

        self._fluid = params['active_fluid']
        # Building hydraulic conductance
        g = self._fluid.get_throat_data(prop='hydraulic_conductance')
        s = self._fluid.get_throat_data(prop='occupancy')
        self._conductance = g*s+g*(-s)/1e3

    def _do_inner_iteration_stage(self):

        p = self._do_one_inner_iteration()
        self._fluid.set_pore_data(prop='pressure',data = p)

<|endoftext|>


<commit_before>from PyQt4 import QtGui

import Orange

from Orange.widgets import gui, widget
from Orange.widgets.settings import Setting

#: Purging flags
SortValues, RemoveConstant, RemoveUnusedValues = 1, 2, 4

class OWPurgeDomain(widget.OWWidget):
    name = "Purge Domain"
    description = "Removes redundant values and attributes, sorts values."
    icon = "icons/PurgeDomain.svg"
    category = "Data"
    keywords = ["data", "purge", "domain"]

    inputs = [("Data", Orange.data.Table, "setData")]
    outputs = [("Data", Orange.data.Table)]

    removeValues = Setting(1)
    removeAttributes = Setting(1)
    removeClassAttribute = Setting(1)
    removeClasses = Setting(1)
    autoSend = Setting(False)
    sortValues = Setting(True)
    sortClasses = Setting(True)

    want_main_area = False

    def __init__(self, parent=None):
        super().__init__(parent)
        self.data = None

        self.preRemoveValues = 1
        self.preRemoveClasses = 1
        self.dataChanged = False

        self.removedAttrs = "-"
        self.reducedAttrs = "-"
        self.resortedAttrs = "-"
        self.classAttr = "-"

        boxAt = gui.widgetBox(self.controlArea, "Attributes", addSpace=True)
        gui.checkBox(boxAt, self, 'sortValues', 'Sort attribute values',
                     callback=self.optionsChanged)
        rua = gui.checkBox(
            boxAt, self, "removeAttributes",
            "Remove attributes with less than two values",
            callback=self.removeAttributesChanged)

        ruv = gui.checkBox(
            gui.indentedBox(boxAt, sep=gui.checkButtonOffsetHint(rua)),
            self,
            "removeValues",
            "Remove unused attribute values",
            callback=self.optionsChanged
        )
        rua.disables = [ruv]
        rua.makeConsistent()

        boxAt = gui.widgetBox(self.controlArea, "Classes", addSpace=True)
        gui.checkBox(boxAt, self, 'sortClasses', 'Sort classes',
                     callback=self.optionsChanged)
        rua = gui.checkBox(
            boxAt, self, "removeClassAttribute",
            "Remove class attribute if there are less than two classes",
            callback=self.removeClassesChanged
        )
        ruv = gui.checkBox(
            gui.indentedBox(boxAt, sep=gui.checkButtonOffsetHint(rua)),
            self,
            "removeClasses",
            "Remove unused class values",
            callback=self.optionsChanged
        )
        rua.disables = [ruv]
        rua.makeConsistent()

        box3 = gui.widgetBox(self.controlArea, 'Statistics', addSpace=True)
        gui.label(box3, self, "Removed attributes: %(removedAttrs)s")
        gui.label(box3, self, "Reduced attributes: %(reducedAttrs)s")
        gui.label(box3, self, "Resorted attributes: %(resortedAttrs)s")
        gui.label(box3, self, "Class attribute: %(classAttr)s")

        box2 = gui.widgetBox(self.controlArea, "Send")
        btSend = gui.button(box2, self, "Send data",
                            callback=self.process,
                            default=True)
        cbAutoSend = gui.checkBox(box2, self, "autoSend", "Send automatically")

        gui.setStopper(self, btSend, cbAutoSend, "dataChanged", self.process)

        gui.rubber(self.controlArea)

    def setData(self, dataset):
        if dataset is not None:
            self.data = dataset
            self.process()
        else:
            self.removedAttrs = "-"
            self.reducedAttrs = "-"
            self.resortedAttrs = "-"
            self.classAttr = "-"
            self.send("Data", None)
            self.data = None
        self.dataChanged = False

    def removeAttributesChanged(self):
        if not self.removeAttributes:
            self.preRemoveValues = self.removeValues
            self.removeValues = False
        else:
            self.removeValues = self.preRemoveValues
        self.optionsChanged()

    def removeClassesChanged(self):
        if not self.removeClassAttribute:
            self.preRemoveClasses = self.removeClasses
            self.removeClasses = False
        else:
            self.removeClasses = self.preRemoveClasses
        self.optionsChanged()

    def optionsChanged(self):
        if self.autoSend:
            self.process()
        else:
            self.dataChanged = True

    def process(self):
        if self.data is None:
            return

        self.reducedAttrs = 0
        self.removedAttrs = 0
        self.resortedAttrs = 0

        attr_flags = sum([SortValues * self.sortValues,
                          RemoveConstant * self.removeAttributes,
                          RemoveUnusedValues * self.removeValues])

        class_flags = sum([SortValues * self.sortClasses,
                           RemoveConstant * self.removeClassAttribute,
                           RemoveUnusedValues * self.removeClasses])
        domain = self.data.domain

        attrs_state = [purge_var_M(var, self.data, attr_flags)
                       for var in domain.attributes]
        class_vars_state = [purge_var_M(var, self.data, class_flags)
                            for var in domain.class_vars]

        nremoved = len([st for st in attrs_state if is_removed(st)])
        nreduced = len([st for st in attrs_state
                        if not is_removed(st) and is_reduced(st)])
        nsorted = len([st for st in attrs_state
                       if not is_removed(st) and is_sorted(st)])

        self.removedAttrs = nremoved
        self.reducedAttrs = nreduced
        self.resortedAttrs = nsorted

        if class_vars_state:
            # TODO: Extend the reporting for multi-class domains
            st = class_vars_state[0]
            if isinstance(st, Var):
                self.classAttr = "Class is unchanged"
            elif is_removed(st):
                self.classAttr = "Class is removed"
            else:
                status = " and ".join(
                    [s for s, predicate in zip(["sorted", "reduced"],
                                               [is_sorted, is_reduced])
                     if predicate(st)]
                )
                self.classAttr = "Class is " + status

        attrs = tuple(merge_transforms(st).var for st in attrs_state
                      if not is_removed(st))
        class_vars = tuple(merge_transforms(st).var for st in class_vars_state
                           if not is_removed(st))

        newdomain = Orange.data.Domain(attrs, class_vars, domain.metas)
        if newdomain.attributes != domain.attributes or \
                newdomain.class_vars != domain.class_vars:
            data = Orange.data.Table.from_table(newdomain, self.data)
        else:
            data = self.data

        self.send("Data", data)

        self.dataChanged = False

import numpy
from collections import namedtuple

# Define a simple Purge expression 'language'.
#: A input variable (leaf expression).
Var = namedtuple("Var", ["var"])
#: Removed variable (can only ever be present as a root node).
Removed = namedtuple("Removed", ["sub", "var"])
#: A reduced variable
Reduced = namedtuple("Reduced", ["sub", "var"])
#: A sorted variable
Sorted = namedtuple("Sorted", ["sub", "var"])
#: A general (lookup) transformed variable.
#: (this node is returned as a result of `merge` which joins consecutive
#: Removed/Reduced nodes into a single Transformed node)
Transformed = namedtuple("Transformed", ["sub", "var"])

def is_var(exp):
    """Is `exp` a `Var` node."""
    return isinstance(exp, Var)

def is_removed(exp):
    """Is `exp` a `Removed` node."""
    return isinstance(exp, Removed)

def _contains(exp, cls):
    """Does `node` contain a sub node of type `cls`"""
    if isinstance(exp, cls):
        return True
    elif isinstance(exp, Var):
        return False
    else:
        return _contains(exp.sub, cls)

def is_reduced(exp):
    """Does `exp` contain a `Reduced` node."""
    return _contains(exp, Reduced)

def is_sorted(exp):
    """Does `exp` contain a `Reduced` node."""
    return _contains(exp, Sorted)

def merge_transforms(exp):
    """
    Merge consecutive Removed, Reduced or Transformed nodes.

    .. note:: Removed nodes are returned unchanged.

    """
    if isinstance(exp, (Var, Removed)):
        return exp
    elif isinstance(exp, (Reduced, Sorted, Transformed)):
        prev = merge_transforms(exp.sub)
        if isinstance(prev, (Reduced, Sorted, Transformed)):
            B = exp.var.get_value_from
            assert isinstance(B, Lookup)
            A = B.variable.get_value_from
            assert isinstance(A, Lookup)

            new_var = Orange.data.DiscreteVariable(
                exp.var.name,
                values=exp.var.values,
                ordered=exp.var.ordered
            )
            new_var.get_value_from = merge_lookup(A, B)
            assert isinstance(prev.sub, Var)
            return Transformed(prev.sub, new_var)
        else:
            assert prev is exp.sub
            return exp
    else:
        raise TypeError

def purge_var_M(var, data, flags):
    state = Var(var)
    if flags & RemoveConstant:
        var = remove_constant(state.var, data)
        if var is None:
            return Removed(state, state.var)

    if isinstance(state.var, Orange.data.DiscreteVariable):
        if flags & RemoveUnusedValues:
            newattr = remove_unused_values(state.var, data)

            if newattr is not state.var:
                state = Reduced(state, newattr)

            if flags & RemoveConstant and len(state.var.values) < 2:
                return Removed(state, state.var)

        if flags & SortValues:
            newattr = sort_var_values(state.var)
            if newattr is not state.var:
                state = Sorted(state, newattr)

    return state

def purge_domain(data, attribute_flags=RemoveConstant | RemoveUnusedValues,
                 class_flags=RemoveConstant | RemoveUnusedValues):

    attrs = [purge_var_M(var, data, attribute_flags)
             for var in data.domain.attributes]
    class_vars = [purge_var_M(var, data, class_flags)
                  for var in data.domain.class_vars]

    attrs = [var for var in attrs if not is_removed(var)]
    class_vars = [var for var in class_vars if not is_removed(var)]
    attrs = [merge_transforms(var).var for var in attrs]
    class_vars = [merge_transforms(var).var for var in class_vars]

    return Orange.data.Domain(attrs, class_vars, data.domain.metas)

def has_at_least_two_values(data, var):
    ((dist, _), ) = data._compute_distributions([var])
    if isinstance(var, Orange.data.ContinuousVariable):
        dist = dist[1, :]
    return numpy.sum(dist > 0.0) > 1

def remove_constant(var, data):
    if isinstance(var, Orange.data.ContinuousVariable):
        if not has_at_least_two_values(data, var):
            return None
        else:
            return var
    elif isinstance(var, Orange.data.DiscreteVariable):
        if len(var.values) < 2:
            return None
        else:
            return var
    else:
        return var

def remove_unused_values(var, data):
    column_data = Orange.data.Table.from_table(
        Orange.data.Domain([var]),
        data
    )
    array = column_data.X.ravel()
    mask = numpy.isfinite(array)
    unique = numpy.array(numpy.unique(array[mask]), dtype=int)

    if len(unique) == len(var.values):
        return var

    used_values = [var.values[i] for i in unique]
    new_var = Orange.data.DiscreteVariable(
        "R_{}".format(var.name),
        values=used_values
    )
    translation_table = numpy.array([numpy.NaN] * len(var.values))
    translation_table[unique] = range(len(new_var.values))

    new_var.get_value_from = Lookup(var, translation_table)
    return new_var

def sort_var_values(var):
    newvalues = list(sorted(var.values))

    if newvalues == list(var.values):
        return var

    translation_table = numpy.array(
        [float(newvalues.index(value)) for value in var.values]
    )

    newvar = Orange.data.DiscreteVariable(var.name, values=newvalues)
    newvar.get_value_from = Lookup(var, translation_table)
    return newvar

from Orange.feature.transformation import Lookup

class Lookup(Lookup):
    def _transform(self, column):
        mask = numpy.isnan(column)
        column_valid = numpy.where(mask, 0, column)
        values = self.lookup_table[numpy.array(column_valid, dtype=int)]
        return numpy.where(mask, numpy.nan, values)

def merge_lookup(A, B):
    """
    Merge two consecutive Lookup transforms into one.
    """
    lookup_table = numpy.array(A.lookup_table)
    mask = numpy.isfinite(lookup_table)
    indices = numpy.array(lookup_table[mask], dtype=int)
    lookup_table[mask] = B.lookup_table[indices]
    return Lookup(A.variable, lookup_table)

if __name__ == "__main__":
    appl = QtGui.QApplication([])
    ow = OWPurgeDomain()
    data = Orange.data.Table("car.tab")
    subset = [inst for inst in data
              if inst["buying"] == "v-high"]
    subset = Orange.data.Table(data.domain, subset)
    # The "buying" should be removed and the class "y" reduced
    ow.setData(subset)
    ow.show()
    appl.exec_()
    ow.saveSettings()
<commit_msg>Added base_value translation when reducing a variable.<commit_after>from PyQt4 import QtGui

import Orange

from Orange.widgets import gui, widget
from Orange.widgets.settings import Setting

#: Purging flags
SortValues, RemoveConstant, RemoveUnusedValues = 1, 2, 4

class OWPurgeDomain(widget.OWWidget):
    name = "Purge Domain"
    description = "Removes redundant values and attributes, sorts values."
    icon = "icons/PurgeDomain.svg"
    category = "Data"
    keywords = ["data", "purge", "domain"]

    inputs = [("Data", Orange.data.Table, "setData")]
    outputs = [("Data", Orange.data.Table)]

    removeValues = Setting(1)
    removeAttributes = Setting(1)
    removeClassAttribute = Setting(1)
    removeClasses = Setting(1)
    autoSend = Setting(False)
    sortValues = Setting(True)
    sortClasses = Setting(True)

    want_main_area = False

    def __init__(self, parent=None):
        super().__init__(parent)
        self.data = None

        self.preRemoveValues = 1
        self.preRemoveClasses = 1
        self.dataChanged = False

        self.removedAttrs = "-"
        self.reducedAttrs = "-"
        self.resortedAttrs = "-"
        self.classAttr = "-"

        boxAt = gui.widgetBox(self.controlArea, "Attributes", addSpace=True)
        gui.checkBox(boxAt, self, 'sortValues', 'Sort attribute values',
                     callback=self.optionsChanged)
        rua = gui.checkBox(
            boxAt, self, "removeAttributes",
            "Remove attributes with less than two values",
            callback=self.removeAttributesChanged)

        ruv = gui.checkBox(
            gui.indentedBox(boxAt, sep=gui.checkButtonOffsetHint(rua)),
            self,
            "removeValues",
            "Remove unused attribute values",
            callback=self.optionsChanged
        )
        rua.disables = [ruv]
        rua.makeConsistent()

        boxAt = gui.widgetBox(self.controlArea, "Classes", addSpace=True)
        gui.checkBox(boxAt, self, 'sortClasses', 'Sort classes',
                     callback=self.optionsChanged)
        rua = gui.checkBox(
            boxAt, self, "removeClassAttribute",
            "Remove class attribute if there are less than two classes",
            callback=self.removeClassesChanged
        )
        ruv = gui.checkBox(
            gui.indentedBox(boxAt, sep=gui.checkButtonOffsetHint(rua)),
            self,
            "removeClasses",
            "Remove unused class values",
            callback=self.optionsChanged
        )
        rua.disables = [ruv]
        rua.makeConsistent()

        box3 = gui.widgetBox(self.controlArea, 'Statistics', addSpace=True)
        gui.label(box3, self, "Removed attributes: %(removedAttrs)s")
        gui.label(box3, self, "Reduced attributes: %(reducedAttrs)s")
        gui.label(box3, self, "Resorted attributes: %(resortedAttrs)s")
        gui.label(box3, self, "Class attribute: %(classAttr)s")

        box2 = gui.widgetBox(self.controlArea, "Send")
        btSend = gui.button(box2, self, "Send data",
                            callback=self.process,
                            default=True)
        cbAutoSend = gui.checkBox(box2, self, "autoSend", "Send automatically")

        gui.setStopper(self, btSend, cbAutoSend, "dataChanged", self.process)

        gui.rubber(self.controlArea)

    def setData(self, dataset):
        if dataset is not None:
            self.data = dataset
            self.process()
        else:
            self.removedAttrs = "-"
            self.reducedAttrs = "-"
            self.resortedAttrs = "-"
            self.classAttr = "-"
            self.send("Data", None)
            self.data = None
        self.dataChanged = False

    def removeAttributesChanged(self):
        if not self.removeAttributes:
            self.preRemoveValues = self.removeValues
            self.removeValues = False
        else:
            self.removeValues = self.preRemoveValues
        self.optionsChanged()

    def removeClassesChanged(self):
        if not self.removeClassAttribute:
            self.preRemoveClasses = self.removeClasses
            self.removeClasses = False
        else:
            self.removeClasses = self.preRemoveClasses
        self.optionsChanged()

    def optionsChanged(self):
        if self.autoSend:
            self.process()
        else:
            self.dataChanged = True

    def process(self):
        if self.data is None:
            return

        self.reducedAttrs = 0
        self.removedAttrs = 0
        self.resortedAttrs = 0

        attr_flags = sum([SortValues * self.sortValues,
                          RemoveConstant * self.removeAttributes,
                          RemoveUnusedValues * self.removeValues])

        class_flags = sum([SortValues * self.sortClasses,
                           RemoveConstant * self.removeClassAttribute,
                           RemoveUnusedValues * self.removeClasses])
        domain = self.data.domain

        attrs_state = [purge_var_M(var, self.data, attr_flags)
                       for var in domain.attributes]
        class_vars_state = [purge_var_M(var, self.data, class_flags)
                            for var in domain.class_vars]

        nremoved = len([st for st in attrs_state if is_removed(st)])
        nreduced = len([st for st in attrs_state
                        if not is_removed(st) and is_reduced(st)])
        nsorted = len([st for st in attrs_state
                       if not is_removed(st) and is_sorted(st)])

        self.removedAttrs = nremoved
        self.reducedAttrs = nreduced
        self.resortedAttrs = nsorted

        if class_vars_state:
            # TODO: Extend the reporting for multi-class domains
            st = class_vars_state[0]
            if isinstance(st, Var):
                self.classAttr = "Class is unchanged"
            elif is_removed(st):
                self.classAttr = "Class is removed"
            else:
                status = " and ".join(
                    [s for s, predicate in zip(["sorted", "reduced"],
                                               [is_sorted, is_reduced])
                     if predicate(st)]
                )
                self.classAttr = "Class is " + status

        attrs = tuple(merge_transforms(st).var for st in attrs_state
                      if not is_removed(st))
        class_vars = tuple(merge_transforms(st).var for st in class_vars_state
                           if not is_removed(st))

        newdomain = Orange.data.Domain(attrs, class_vars, domain.metas)
        if newdomain.attributes != domain.attributes or \
                newdomain.class_vars != domain.class_vars:
            data = Orange.data.Table.from_table(newdomain, self.data)
        else:
            data = self.data

        self.send("Data", data)

        self.dataChanged = False

import numpy
from collections import namedtuple

# Define a simple Purge expression 'language'.
#: A input variable (leaf expression).
Var = namedtuple("Var", ["var"])
#: Removed variable (can only ever be present as a root node).
Removed = namedtuple("Removed", ["sub", "var"])
#: A reduced variable
Reduced = namedtuple("Reduced", ["sub", "var"])
#: A sorted variable
Sorted = namedtuple("Sorted", ["sub", "var"])
#: A general (lookup) transformed variable.
#: (this node is returned as a result of `merge` which joins consecutive
#: Removed/Reduced nodes into a single Transformed node)
Transformed = namedtuple("Transformed", ["sub", "var"])

def is_var(exp):
    """Is `exp` a `Var` node."""
    return isinstance(exp, Var)

def is_removed(exp):
    """Is `exp` a `Removed` node."""
    return isinstance(exp, Removed)

def _contains(exp, cls):
    """Does `node` contain a sub node of type `cls`"""
    if isinstance(exp, cls):
        return True
    elif isinstance(exp, Var):
        return False
    else:
        return _contains(exp.sub, cls)

def is_reduced(exp):
    """Does `exp` contain a `Reduced` node."""
    return _contains(exp, Reduced)

def is_sorted(exp):
    """Does `exp` contain a `Reduced` node."""
    return _contains(exp, Sorted)

def merge_transforms(exp):
    """
    Merge consecutive Removed, Reduced or Transformed nodes.

    .. note:: Removed nodes are returned unchanged.

    """
    if isinstance(exp, (Var, Removed)):
        return exp
    elif isinstance(exp, (Reduced, Sorted, Transformed)):
        prev = merge_transforms(exp.sub)
        if isinstance(prev, (Reduced, Sorted, Transformed)):
            B = exp.var.get_value_from
            assert isinstance(B, Lookup)
            A = B.variable.get_value_from
            assert isinstance(A, Lookup)

            new_var = Orange.data.DiscreteVariable(
                exp.var.name,
                values=exp.var.values,
                ordered=exp.var.ordered
            )
            new_var.get_value_from = merge_lookup(A, B)
            assert isinstance(prev.sub, Var)
            return Transformed(prev.sub, new_var)
        else:
            assert prev is exp.sub
            return exp
    else:
        raise TypeError

def purge_var_M(var, data, flags):
    state = Var(var)
    if flags & RemoveConstant:
        var = remove_constant(state.var, data)
        if var is None:
            return Removed(state, state.var)

    if isinstance(state.var, Orange.data.DiscreteVariable):
        if flags & RemoveUnusedValues:
            newattr = remove_unused_values(state.var, data)

            if newattr is not state.var:
                state = Reduced(state, newattr)

            if flags & RemoveConstant and len(state.var.values) < 2:
                return Removed(state, state.var)

        if flags & SortValues:
            newattr = sort_var_values(state.var)
            if newattr is not state.var:
                state = Sorted(state, newattr)

    return state

def purge_domain(data, attribute_flags=RemoveConstant | RemoveUnusedValues,
                 class_flags=RemoveConstant | RemoveUnusedValues):

    attrs = [purge_var_M(var, data, attribute_flags)
             for var in data.domain.attributes]
    class_vars = [purge_var_M(var, data, class_flags)
                  for var in data.domain.class_vars]

    attrs = [var for var in attrs if not is_removed(var)]
    class_vars = [var for var in class_vars if not is_removed(var)]
    attrs = [merge_transforms(var).var for var in attrs]
    class_vars = [merge_transforms(var).var for var in class_vars]

    return Orange.data.Domain(attrs, class_vars, data.domain.metas)

def has_at_least_two_values(data, var):
    ((dist, _), ) = data._compute_distributions([var])
    if isinstance(var, Orange.data.ContinuousVariable):
        dist = dist[1, :]
    return numpy.sum(dist > 0.0) > 1

def remove_constant(var, data):
    if isinstance(var, Orange.data.ContinuousVariable):
        if not has_at_least_two_values(data, var):
            return None
        else:
            return var
    elif isinstance(var, Orange.data.DiscreteVariable):
        if len(var.values) < 2:
            return None
        else:
            return var
    else:
        return var

def remove_unused_values(var, data):
    column_data = Orange.data.Table.from_table(
        Orange.data.Domain([var]),
        data
    )
    array = column_data.X.ravel()
    mask = numpy.isfinite(array)
    unique = numpy.array(numpy.unique(array[mask]), dtype=int)

    if len(unique) == len(var.values):
        return var

    used_values = [var.values[i] for i in unique]
    new_var = Orange.data.DiscreteVariable(
        "R_{}".format(var.name),
        values=used_values
    )
    translation_table = numpy.array([numpy.NaN] * len(var.values))
    translation_table[unique] = range(len(new_var.values))

    if 0 >= var.base_value < len(var.values):
        base = translation_table[var.base_value]
        if numpy.isfinite(base):
            new_var.base_value = int(base)

    new_var.get_value_from = Lookup(var, translation_table)
    return new_var

def sort_var_values(var):
    newvalues = list(sorted(var.values))

    if newvalues == list(var.values):
        return var

    translation_table = numpy.array(
        [float(newvalues.index(value)) for value in var.values]
    )

    newvar = Orange.data.DiscreteVariable(var.name, values=newvalues)
    newvar.get_value_from = Lookup(var, translation_table)
    return newvar

from Orange.feature.transformation import Lookup

class Lookup(Lookup):
    def _transform(self, column):
        mask = numpy.isnan(column)
        column_valid = numpy.where(mask, 0, column)
        values = self.lookup_table[numpy.array(column_valid, dtype=int)]
        return numpy.where(mask, numpy.nan, values)

def merge_lookup(A, B):
    """
    Merge two consecutive Lookup transforms into one.
    """
    lookup_table = numpy.array(A.lookup_table)
    mask = numpy.isfinite(lookup_table)
    indices = numpy.array(lookup_table[mask], dtype=int)
    lookup_table[mask] = B.lookup_table[indices]
    return Lookup(A.variable, lookup_table)

if __name__ == "__main__":
    appl = QtGui.QApplication([])
    ow = OWPurgeDomain()
    data = Orange.data.Table("car.tab")
    subset = [inst for inst in data
              if inst["buying"] == "v-high"]
    subset = Orange.data.Table(data.domain, subset)
    # The "buying" should be removed and the class "y" reduced
    ow.setData(subset)
    ow.show()
    appl.exec_()
    ow.saveSettings()
<|endoftext|>


<commit_before>import os
import re

attribute_regex = re.compile(r"\battributes\.(\w+)\b(?!\s*\()", re.I)
end_tag_regex = re.compile(
    r"thistag\.executionmode\s+(?:eq|is|==)\s+[\"']end[\"']", re.I
)
alt_end_tag_regex = re.compile(r"<cfcase\s+value=\s*[\"']end[\"']\s*>")

def index(custom_tag_path):
    custom_tags = {}
    for path in os.listdir(custom_tag_path):
        if path.endswith(".cfm") or path.endswith(".cfc"):
            full_file_path = custom_tag_path.replace("\\", "/") + "/" + path
            file_index = index_file(full_file_path)
            if file_index:
                custom_tags[full_file_path] = file_index
    return custom_tags

def index_file(full_file_path):
    try:
        with open(full_file_path, "r", encoding="utf-8") as f:
            file_string = f.read()
    except Exception:
        print("CFML: unable to read file - " + full_file_path)
        return None

    file_index = {"tag_name": full_file_path.split("/").pop()[:-4]}
    file_index.update(parse_cfm_file_string(file_string))
    return file_index

def parse_cfm_file_string(file_string):
    tag_index = {"has_end_tag": False}
    tag_index["attributes"] = list(
        sorted(set([attr.lower() for attr in re.findall(attribute_regex, file_string)]))
    )
    if re.search(end_tag_regex, file_string):
        tag_index["has_end_tag"] = True
    elif re.search(alt_end_tag_regex, file_string):
        tag_index["has_end_tag"] = True
    return tag_index
<commit_msg>Make custom tag folder search recursive<commit_after>import os
import re

attribute_regex = re.compile(r"\battributes\.(\w+)\b(?!\s*\()", re.I)
end_tag_regex = re.compile(
    r"thistag\.executionmode\s+(?:eq|is|==)\s+[\"']end[\"']", re.I
)
alt_end_tag_regex = re.compile(r"<cfcase\s+value=\s*[\"']end[\"']\s*>")

def index(custom_tag_path):
    custom_tags = {}
    for path, directories, filenames in os.walk(custom_tag_path):
        for filename in filenames:
            if filename.endswith(".cfm") or filename.endswith(".cfc"):
                full_file_path = path.replace("\\", "/") + "/" + filename
                file_index = index_file(full_file_path)
                if file_index:
                    custom_tags[full_file_path] = file_index
    return custom_tags

def index_file(full_file_path):
    try:
        with open(full_file_path, "r", encoding="utf-8") as f:
            file_string = f.read()
    except Exception:
        print("CFML: unable to read file - " + full_file_path)
        return None

    file_index = {"tag_name": full_file_path.split("/").pop()[:-4]}
    file_index.update(parse_cfm_file_string(file_string))
    return file_index

def parse_cfm_file_string(file_string):
    tag_index = {"has_end_tag": False}
    tag_index["attributes"] = list(
        sorted(set([attr.lower() for attr in re.findall(attribute_regex, file_string)]))
    )
    if re.search(end_tag_regex, file_string):
        tag_index["has_end_tag"] = True
    elif re.search(alt_end_tag_regex, file_string):
        tag_index["has_end_tag"] = True
    return tag_index
<|endoftext|>


<commit_before>import pytz
import uuid
from corehq.apps.casegroups.models import CommCareCaseGroup
from corehq.apps.groups.models import Group
from corehq.apps.locations.dbaccessors import get_all_users_by_location
from corehq.apps.locations.models import SQLLocation
from corehq.apps.users.models import CommCareUser, WebUser
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors
from corehq.messaging.scheduling.exceptions import UnknownRecipientType
from corehq.messaging.scheduling.models import AlertSchedule, TimedSchedule
from corehq.sql_db.models import PartitionedModel
from corehq.util.timezones.utils import get_timezone_for_domain, coerce_timezone_value
from datetime import tzinfo
from dimagi.utils.decorators.memoized import memoized
from django.db import models
from django.core.exceptions import ValidationError

class ScheduleInstance(PartitionedModel):
    schedule_instance_id = models.UUIDField(primary_key=True, default=uuid.uuid4)
    domain = models.CharField(max_length=126)
    recipient_type = models.CharField(max_length=126)
    recipient_id = models.CharField(max_length=126, null=True)
    current_event_num = models.IntegerField()
    schedule_iteration_num = models.IntegerField()
    next_event_due = models.DateTimeField()
    active = models.BooleanField()

    class Meta:
        abstract = True
        index_together = (
            # index for equality comparisons on the leading columns
            ('active', 'next_event_due'),
            ('domain', 'active', 'next_event_due'),
        )

    @property
    @memoized
    def recipient(self):
        if self.recipient_type == 'CommCareCase':
            return CaseAccessors(self.domain).get_case(self.recipient_id)
        elif self.recipient_type == 'CommCareUser':
            return CommCareUser.get(self.recipient_id)
        elif self.recipient_type == 'WebUser':
            return WebUser.get(self.recipient_id)
        elif self.recipient_type == 'CommCareCaseGroup':
            return CommCareCaseGroup.get(self.recipient_id)
        elif self.recipient_type == 'Group':
            return Group.get(self.recipient_id)
        elif self.recipient_type == 'Location':
            return SQLLocation.by_location_id(self.recipient_id)
        else:
            raise UnknownRecipientType(self.recipient_type)

    @property
    @memoized
    def timezone(self):
        timezone = None

        if self.recipient_type in ('CommCareCase', 'CommCareUser', 'WebUser'):
            try:
                timezone = self.recipient.get_time_zone()
            except ValidationError:
                pass

        if not timezone:
            timezone = get_timezone_for_domain(self.domain)

        if isinstance(timezone, tzinfo):
            return timezone

        if isinstance(timezone, basestring):
            try:
                return coerce_timezone_value(timezone)
            except ValidationError:
                pass

        return pytz.UTC

    @classmethod
    def create_for_recipient(cls, schedule, recipient_type, recipient_id, start_date=None,
            move_to_next_event_not_in_the_past=True, **additional_fields):

        obj = cls(
            domain=schedule.domain,
            recipient_type=recipient_type,
            recipient_id=recipient_id,
            current_event_num=0,
            schedule_iteration_num=1,
            active=True,
            **additional_fields
        )

        obj.schedule = schedule
        schedule.set_first_event_due_timestamp(obj, start_date)

        if move_to_next_event_not_in_the_past:
            schedule.move_to_next_event_not_in_the_past(obj)

        return obj

    def expand_recipients(self):
        """
        Can be used as a generator to iterate over all individual contacts who
        are the recipients of this ScheduleInstance.
        """
        if self.recipient_type in ('CommCareCase', 'CommCareUser', 'WebUser'):
            yield self.recipient
        elif self.recipient_type == 'CommCareCaseGroup':
            case_group = self.recipient
            for case in case_group.get_cases():
                yield case
        elif self.recipient_type == 'Group':
            group = self.recipient
            for user in group.get_users(is_active=True, only_commcare=False):
                yield user
        elif self.recipient_type == 'Location':
            location = self.recipient
            if self.memoized_schedule.include_descendant_locations:
                location_ids = location.get_descendants(include_self=True).filter(is_archived=False).location_ids()
            else:
                location_ids = [location.location_id]

            user_ids = set()
            for location_id in location_ids:
                for user in get_all_users_by_location(self.domain, location_id):
                    if user.is_active and user.get_id not in user_ids:
                        user_ids.add(user.get_id)
                        yield user
        else:
            raise UnknownRecipientType(self.recipient_type)

    def handle_current_event(self):
        content = self.memoized_schedule.get_current_event_content(self)
        for recipient in self.expand_recipients():
            content.send(recipient)
        # As a precaution, always explicitly move to the next event after processing the current
        # event to prevent ever getting stuck on the current event.
        self.memoized_schedule.move_to_next_event(self)
        self.memoized_schedule.move_to_next_event_not_in_the_past(self)

    @property
    def schedule(self):
        raise NotImplementedError()

    @schedule.setter
    def schedule(self, value):
        raise NotImplementedError()

    @property
    @memoized
    def memoized_schedule(self):
        """
        This is named with a memoized_ prefix to be clear that it should only be used
        when the schedule is not changing.
        """
        return self.schedule

class AbstractAlertScheduleInstance(ScheduleInstance):
    alert_schedule_id = models.UUIDField()

    class Meta(ScheduleInstance.Meta):
        abstract = True

    @property
    def schedule(self):
        return AlertSchedule.objects.get(schedule_id=self.alert_schedule_id)

    @schedule.setter
    def schedule(self, value):
        if not isinstance(value, AlertSchedule):
            raise ValueError("Expected an instance of AlertSchedule")

        self.alert_schedule_id = value.schedule_id

class AbstractTimedScheduleInstance(ScheduleInstance):
    timed_schedule_id = models.UUIDField()
    start_date = models.DateField()

    class Meta(ScheduleInstance.Meta):
        abstract = True

    @property
    def schedule(self):
        return TimedSchedule.objects.get(schedule_id=self.timed_schedule_id)

    @schedule.setter
    def schedule(self, value):
        if not isinstance(value, TimedSchedule):
            raise ValueError("Expected an instance of TimedSchedule")

        self.timed_schedule_id = value.schedule_id

    def recalculate_schedule(self, schedule=None, new_start_date=None):
        schedule = schedule or self.memoized_schedule
        self.current_event_num = 0
        self.schedule_iteration_num = 1
        self.active = True
        if new_start_date:
            self.start_date = new_start_date
        schedule.set_first_event_due_timestamp(self, self.start_date)
        schedule.move_to_next_event_not_in_the_past(self)

class AlertScheduleInstance(AbstractAlertScheduleInstance):

    class Meta(AbstractAlertScheduleInstance.Meta):
        db_table = 'scheduling_alertscheduleinstance'

class TimedScheduleInstance(AbstractTimedScheduleInstance):

    class Meta(AbstractTimedScheduleInstance.Meta):
        db_table = 'scheduling_timedscheduleinstance'

class CaseScheduleInstanceMixin(object):

    @property
    @memoized
    def recipient(self):
        if self.recipient_type == 'Self':
            return CaseAccessors(self.domain).get_case(self.case_id)
        elif self.recipient_type == 'Owner':
            return None
        if self.recipient_type == 'LastSubmittingUser':
            return None
        elif self.recipient_type == 'ParentCase':
            return None
        elif self.recipient_type == 'SubCase':
            return None
        elif self.recipient_type == 'CustomRecipient':
            return None
        else:
            return super(CaseScheduleInstanceMixin, self).recipient

class CaseAlertScheduleInstance(CaseScheduleInstanceMixin, AbstractAlertScheduleInstance):
    # Points to the CommCareCase/SQL that spawned this schedule instance
    case_id = models.CharField(max_length=255)

    # Points to the AutomaticUpdateRule that spawned this schedule instance
    rule_id = models.IntegerField()

    class Meta(AbstractAlertScheduleInstance.Meta):
        db_table = 'scheduling_casealertscheduleinstance'
        index_together = AbstractAlertScheduleInstance.Meta.index_together + (
            ('case_id', 'alert_schedule_id'),
        )

class CaseTimedScheduleInstance(CaseScheduleInstanceMixin, AbstractTimedScheduleInstance):
    # Points to the CommCareCase/SQL that spawned this schedule instance
    case_id = models.CharField(max_length=255)

    # Points to the AutomaticUpdateRule that spawned this schedule instance
    rule_id = models.IntegerField()

    class Meta(AbstractTimedScheduleInstance.Meta):
        db_table = 'scheduling_casetimedscheduleinstance'
        index_together = AbstractTimedScheduleInstance.Meta.index_together + (
            ('case_id', 'timed_schedule_id'),
        )
<commit_msg>Check object type rather than recipient_type<commit_after>import pytz
import uuid
from corehq.apps.casegroups.models import CommCareCaseGroup
from corehq.apps.groups.models import Group
from corehq.apps.locations.dbaccessors import get_all_users_by_location
from corehq.apps.locations.models import SQLLocation
from corehq.apps.users.models import CommCareUser, WebUser
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors
from corehq.form_processor.utils import is_commcarecase
from corehq.messaging.scheduling.exceptions import UnknownRecipientType
from corehq.messaging.scheduling.models import AlertSchedule, TimedSchedule
from corehq.sql_db.models import PartitionedModel
from corehq.util.timezones.utils import get_timezone_for_domain, coerce_timezone_value
from datetime import tzinfo
from dimagi.utils.decorators.memoized import memoized
from django.db import models
from django.core.exceptions import ValidationError

class ScheduleInstance(PartitionedModel):
    schedule_instance_id = models.UUIDField(primary_key=True, default=uuid.uuid4)
    domain = models.CharField(max_length=126)
    recipient_type = models.CharField(max_length=126)
    recipient_id = models.CharField(max_length=126, null=True)
    current_event_num = models.IntegerField()
    schedule_iteration_num = models.IntegerField()
    next_event_due = models.DateTimeField()
    active = models.BooleanField()

    class Meta:
        abstract = True
        index_together = (
            # index for equality comparisons on the leading columns
            ('active', 'next_event_due'),
            ('domain', 'active', 'next_event_due'),
        )

    @property
    @memoized
    def recipient(self):
        if self.recipient_type == 'CommCareCase':
            return CaseAccessors(self.domain).get_case(self.recipient_id)
        elif self.recipient_type == 'CommCareUser':
            return CommCareUser.get(self.recipient_id)
        elif self.recipient_type == 'WebUser':
            return WebUser.get(self.recipient_id)
        elif self.recipient_type == 'CommCareCaseGroup':
            return CommCareCaseGroup.get(self.recipient_id)
        elif self.recipient_type == 'Group':
            return Group.get(self.recipient_id)
        elif self.recipient_type == 'Location':
            return SQLLocation.by_location_id(self.recipient_id)
        else:
            raise UnknownRecipientType(self.recipient_type)

    @property
    @memoized
    def recipient_is_an_individual_contact(self):
        return (
            isinstance(self.recipient, (CommCareUser, WebUser)) or
            is_commcarecase(self.recipient)
        )

    @property
    @memoized
    def timezone(self):
        timezone = None

        if self.recipient_is_an_individual_contact:
            try:
                timezone = self.recipient.get_time_zone()
            except ValidationError:
                pass

        if not timezone:
            timezone = get_timezone_for_domain(self.domain)

        if isinstance(timezone, tzinfo):
            return timezone

        if isinstance(timezone, basestring):
            try:
                return coerce_timezone_value(timezone)
            except ValidationError:
                pass

        return pytz.UTC

    @classmethod
    def create_for_recipient(cls, schedule, recipient_type, recipient_id, start_date=None,
            move_to_next_event_not_in_the_past=True, **additional_fields):

        obj = cls(
            domain=schedule.domain,
            recipient_type=recipient_type,
            recipient_id=recipient_id,
            current_event_num=0,
            schedule_iteration_num=1,
            active=True,
            **additional_fields
        )

        obj.schedule = schedule
        schedule.set_first_event_due_timestamp(obj, start_date)

        if move_to_next_event_not_in_the_past:
            schedule.move_to_next_event_not_in_the_past(obj)

        return obj

    def expand_recipients(self):
        """
        Can be used as a generator to iterate over all individual contacts who
        are the recipients of this ScheduleInstance.
        """
        if self.recipient_is_an_individual_contact:
            yield self.recipient
        elif isinstance(self.recipient, CommCareCaseGroup):
            case_group = self.recipient
            for case in case_group.get_cases():
                yield case
        elif isinstance(self.recipient, Group):
            group = self.recipient
            for user in group.get_users(is_active=True, only_commcare=False):
                yield user
        elif isinstance(self.recipient, SQLLocation):
            location = self.recipient
            if self.memoized_schedule.include_descendant_locations:
                location_ids = location.get_descendants(include_self=True).filter(is_archived=False).location_ids()
            else:
                location_ids = [location.location_id]

            user_ids = set()
            for location_id in location_ids:
                for user in get_all_users_by_location(self.domain, location_id):
                    if user.is_active and user.get_id not in user_ids:
                        user_ids.add(user.get_id)
                        yield user
        else:
            raise UnknownRecipientType(self.recipient.__class__.__name__)

    def handle_current_event(self):
        content = self.memoized_schedule.get_current_event_content(self)
        for recipient in self.expand_recipients():
            content.send(recipient)
        # As a precaution, always explicitly move to the next event after processing the current
        # event to prevent ever getting stuck on the current event.
        self.memoized_schedule.move_to_next_event(self)
        self.memoized_schedule.move_to_next_event_not_in_the_past(self)

    @property
    def schedule(self):
        raise NotImplementedError()

    @schedule.setter
    def schedule(self, value):
        raise NotImplementedError()

    @property
    @memoized
    def memoized_schedule(self):
        """
        This is named with a memoized_ prefix to be clear that it should only be used
        when the schedule is not changing.
        """
        return self.schedule

class AbstractAlertScheduleInstance(ScheduleInstance):
    alert_schedule_id = models.UUIDField()

    class Meta(ScheduleInstance.Meta):
        abstract = True

    @property
    def schedule(self):
        return AlertSchedule.objects.get(schedule_id=self.alert_schedule_id)

    @schedule.setter
    def schedule(self, value):
        if not isinstance(value, AlertSchedule):
            raise ValueError("Expected an instance of AlertSchedule")

        self.alert_schedule_id = value.schedule_id

class AbstractTimedScheduleInstance(ScheduleInstance):
    timed_schedule_id = models.UUIDField()
    start_date = models.DateField()

    class Meta(ScheduleInstance.Meta):
        abstract = True

    @property
    def schedule(self):
        return TimedSchedule.objects.get(schedule_id=self.timed_schedule_id)

    @schedule.setter
    def schedule(self, value):
        if not isinstance(value, TimedSchedule):
            raise ValueError("Expected an instance of TimedSchedule")

        self.timed_schedule_id = value.schedule_id

    def recalculate_schedule(self, schedule=None, new_start_date=None):
        schedule = schedule or self.memoized_schedule
        self.current_event_num = 0
        self.schedule_iteration_num = 1
        self.active = True
        if new_start_date:
            self.start_date = new_start_date
        schedule.set_first_event_due_timestamp(self, self.start_date)
        schedule.move_to_next_event_not_in_the_past(self)

class AlertScheduleInstance(AbstractAlertScheduleInstance):

    class Meta(AbstractAlertScheduleInstance.Meta):
        db_table = 'scheduling_alertscheduleinstance'

class TimedScheduleInstance(AbstractTimedScheduleInstance):

    class Meta(AbstractTimedScheduleInstance.Meta):
        db_table = 'scheduling_timedscheduleinstance'

class CaseScheduleInstanceMixin(object):

    @property
    @memoized
    def recipient(self):
        if self.recipient_type == 'Self':
            return CaseAccessors(self.domain).get_case(self.case_id)
        elif self.recipient_type == 'Owner':
            return None
        if self.recipient_type == 'LastSubmittingUser':
            return None
        elif self.recipient_type == 'ParentCase':
            return None
        elif self.recipient_type == 'SubCase':
            return None
        elif self.recipient_type == 'CustomRecipient':
            return None
        else:
            return super(CaseScheduleInstanceMixin, self).recipient

class CaseAlertScheduleInstance(CaseScheduleInstanceMixin, AbstractAlertScheduleInstance):
    # Points to the CommCareCase/SQL that spawned this schedule instance
    case_id = models.CharField(max_length=255)

    # Points to the AutomaticUpdateRule that spawned this schedule instance
    rule_id = models.IntegerField()

    class Meta(AbstractAlertScheduleInstance.Meta):
        db_table = 'scheduling_casealertscheduleinstance'
        index_together = AbstractAlertScheduleInstance.Meta.index_together + (
            ('case_id', 'alert_schedule_id'),
        )

class CaseTimedScheduleInstance(CaseScheduleInstanceMixin, AbstractTimedScheduleInstance):
    # Points to the CommCareCase/SQL that spawned this schedule instance
    case_id = models.CharField(max_length=255)

    # Points to the AutomaticUpdateRule that spawned this schedule instance
    rule_id = models.IntegerField()

    class Meta(AbstractTimedScheduleInstance.Meta):
        db_table = 'scheduling_casetimedscheduleinstance'
        index_together = AbstractTimedScheduleInstance.Meta.index_together + (
            ('case_id', 'timed_schedule_id'),
        )
<|endoftext|>