<commit_before># Copyright (C) 2017 Linaro Limited
#
# Author: Remi Duraffort <remi.duraffort@linaro.org>
#
# This file is part of LAVA Dispatcher.
#
# LAVA Dispatcher is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# LAVA Dispatcher is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along
# with this program; if not, see <http://www.gnu.org/licenses>.

# pylint: disable=wrong-import-order

import logging
import os
import time
import yaml
import zmq
import zmq.auth
from zmq.utils.strtypes import u
from zmq.auth.thread import ThreadAuthenticator

from django.db import connection, transaction
from django.db.utils import OperationalError, InterfaceError

from lava_results_app.models import TestCase
from lava_server.cmdutils import LAVADaemonCommand, watch_directory
from lava_scheduler_app.models import TestJob
from lava_scheduler_app.utils import mkdir
from lava_results_app.dbutils import map_scanned_results, create_metadata_store

# Constants
FORMAT = "%(asctime)-15s %(levelname)7s %(message)s"
TIMEOUT = 10
BULK_CREATE_TIMEOUT = 10
FD_TIMEOUT = 60

class JobHandler(object):  # pylint: disable=too-few-public-methods
    def __init__(self, job):
        self.output_dir = job.output_dir
        self.output = open(os.path.join(self.output_dir, 'output.yaml'), 'a+')
        self.last_usage = time.time()

    def write(self, message):
        self.output.write(message)
        self.output.write('\n')
        self.output.flush()

    def close(self):
        self.output.close()

class Command(LAVADaemonCommand):
    help = "LAVA log recorder"
    logger = None
    default_logfile = "/var/log/lava-server/lava-logs.log"

    def __init__(self, *args, **options):
        super(Command, self).__init__(*args, **options)
        self.logger = logging.getLogger("lava-logs")
        self.log_socket = None
        self.auth = None
        self.controler = None
        self.inotify_fd = None
        self.pipe_r = None
        self.poller = None
        self.cert_dir_path = None
        # List of logs
        self.jobs = {}
        # Keep test cases in memory
        self.test_cases = []
        # Master status
        self.last_ping = 0
        self.ping_interval = TIMEOUT

    def add_arguments(self, parser):
        super(Command, self).add_arguments(parser)

        net = parser.add_argument_group("network")
        net.add_argument('--socket',
                         default='tcp://*:5555',
                         help="Socket waiting for logs. Default: tcp://*:5555")
        net.add_argument('--master-socket',
                         default='tcp://localhost:5556',
                         help="Socket for master-slave communication. Default: tcp://localhost:5556")
        net.add_argument('--ipv6', default=False, action='store_true',
                         help="Enable IPv6 on the listening sockets")
        net.add_argument('--encrypt', default=False, action='store_true',
                         help="Encrypt messages")
        net.add_argument('--master-cert',
                         default='/etc/lava-dispatcher/certificates.d/master.key_secret',
                         help="Certificate for the master socket")
        net.add_argument('--slaves-certs',
                         default='/etc/lava-dispatcher/certificates.d',
                         help="Directory for slaves certificates")

    def handle(self, *args, **options):
        # Initialize logging.
        self.setup_logging("lava-logs", options["level"],
                           options["log_file"], FORMAT)

        self.logger.info("[INIT] Dropping privileges")
        if not self.drop_privileges(options['user'], options['group']):
            self.logger.error("[INIT] Unable to drop privileges")
            return

        # Create the sockets
        context = zmq.Context()
        self.log_socket = context.socket(zmq.PULL)
        self.controler = context.socket(zmq.ROUTER)
        self.controler.setsockopt(zmq.IDENTITY, b"lava-logs")
        # Limit the number of messages in the queue
        self.controler.setsockopt(zmq.SNDHWM, 2)
        # TODO: remove when Jessie is not supported
        if hasattr(zmq, "CONNECT_RID"):
            # From http://api.zeromq.org/4-2:zmq-setsockopt#toc5
            # "Immediately readies that connection for data transfer with the master"
            self.controler.setsockopt(zmq.CONNECT_RID, b"master")

        if options['ipv6']:
            self.logger.info("[INIT] Enabling IPv6")
            self.log_socket.setsockopt(zmq.IPV6, 1)
            self.controler.setsockopt(zmq.IPV6, 1)

        if options['encrypt']:
            self.logger.info("[INIT] Starting encryption")
            try:
                self.auth = ThreadAuthenticator(context)
                self.auth.start()
                self.logger.debug("[INIT] Opening master certificate: %s", options['master_cert'])
                master_public, master_secret = zmq.auth.load_certificate(options['master_cert'])
                self.logger.debug("[INIT] Using slaves certificates from: %s", options['slaves_certs'])
                self.auth.configure_curve(domain='*', location=options['slaves_certs'])
            except IOError as err:
                self.logger.error("[INIT] %s", err)
                self.auth.stop()
                return
            self.log_socket.curve_publickey = master_public
            self.log_socket.curve_secretkey = master_secret
            self.log_socket.curve_server = True
            self.controler.curve_publickey = master_public
            self.controler.curve_secretkey = master_secret
            self.controler.curve_serverkey = master_public

        self.logger.debug("[INIT] Watching %s", options["slaves_certs"])
        self.cert_dir_path = options["slaves_certs"]
        self.inotify_fd = watch_directory(options["slaves_certs"])
        if self.inotify_fd is None:
            self.logger.error("[INIT] Unable to start inotify")

        self.log_socket.bind(options['socket'])
        self.controler.connect(options['master_socket'])

        # Poll on the sockets. This allow to have a
        # nice timeout along with polling.
        self.poller = zmq.Poller()
        self.poller.register(self.log_socket, zmq.POLLIN)
        self.poller.register(self.controler, zmq.POLLIN)
        if self.inotify_fd is not None:
            self.poller.register(os.fdopen(self.inotify_fd), zmq.POLLIN)

        # Translate signals into zmq messages
        (self.pipe_r, _) = self.setup_zmq_signal_handler()
        self.poller.register(self.pipe_r, zmq.POLLIN)

        self.logger.info("[INIT] listening for logs")
        # PING right now: the master is waiting for this message to start
        # scheduling.
        self.controler.send_multipart([b"master", b"PING"])

        try:
            self.main_loop()
        except BaseException as exc:
            self.logger.error("[EXIT] Unknown exception raised, leaving!")
            self.logger.exception(exc)

        # Close the controler socket
        self.controler.close(linger=0)
        self.poller.unregister(self.controler)

        # Carefully close the logging socket as we don't want to lose messages
        self.logger.info("[EXIT] Disconnect logging socket and process messages")
        endpoint = u(self.log_socket.getsockopt(zmq.LAST_ENDPOINT))
        self.logger.debug("[EXIT] unbinding from '%s'", endpoint)
        self.log_socket.unbind(endpoint)

        # Empty the queue
        try:
            while self.wait_for_messages(True):
                # Flush test cases cache for every iteration because we might
                # get killed soon.
                self.flush_test_cases()
        except BaseException as exc:
            self.logger.error("[EXIT] Unknown exception raised, leaving!")
            self.logger.exception(exc)
        finally:
            # Last flush
            self.flush_test_cases()
            self.logger.info("[EXIT] Closing the logging socket: the queue is empty")
            self.log_socket.close()
            if options['encrypt']:
                self.auth.stop()
            context.term()

    def flush_test_cases(self):
        if self.test_cases:
            self.logger.info("Saving %d test cases", len(self.test_cases))
            TestCase.objects.bulk_create(self.test_cases)
            self.test_cases = []

    def main_loop(self):
        last_gc = time.time()
        last_bulk_create = time.time()

        # Wait for messages
        # TODO: fix timeout computation
        while self.wait_for_messages(False):
            now = time.time()

            # Dump TestCase into the database
            if now - last_bulk_create > BULK_CREATE_TIMEOUT:
                last_bulk_create = now
                self.flush_test_cases()

            # Close old file handlers
            if now - last_gc > FD_TIMEOUT:
                last_gc = now
                # Iterate while removing keys is not compatible with iterator
                for job_id in list(self.jobs.keys()):  # pylint: disable=consider-iterating-dictionary
                    if now - self.jobs[job_id].last_usage > FD_TIMEOUT:
                        self.logger.info("[%s] closing log file", job_id)
                        self.jobs[job_id].close()
                        del self.jobs[job_id]

            # Ping the master
            if now - self.last_ping > self.ping_interval:
                self.logger.debug("PING => master")
                self.last_ping = now
                self.controler.send_multipart([b"master", b"PING"])

    def wait_for_messages(self, leaving):
        try:
            try:
                sockets = dict(self.poller.poll(TIMEOUT * 1000))
            except zmq.error.ZMQError as exc:
                self.logger.error("[POLL] zmq error: %s", str(exc))
                return True

            # Messages
            if sockets.get(self.log_socket) == zmq.POLLIN:
                self.logging_socket()
                return True

            # Signals
            elif sockets.get(self.pipe_r) == zmq.POLLIN:
                # remove the message from the queue
                os.read(self.pipe_r, 1)

                if not leaving:
                    self.logger.info("[POLL] received a signal, leaving")
                    return False
                else:
                    self.logger.warning("[POLL] signal already handled, please wait for the process to exit")
                    return True

            # Pong received
            elif sockets.get(self.controler) == zmq.POLLIN:
                self.controler_socket()
                return True

            # Inotify socket
            if sockets.get(self.inotify_fd) == zmq.POLLIN:
                os.read(self.inotify_fd, 4096)
                self.logger.debug("[AUTH] Reloading certificates from %s",
                                  self.cert_dir_path)
                self.auth.configure_curve(domain='*',
                                          location=self.cert_dir_path)

            # Nothing received
            else:
                return not leaving

        except (OperationalError, InterfaceError):
            self.logger.info("[RESET] database connection reset")
            connection.close()
        return True

    def logging_socket(self):
        msg = self.log_socket.recv_multipart()
        try:
            (job_id, message) = (u(m) for m in msg)  # pylint: disable=unbalanced-tuple-unpacking
        except ValueError:
            # do not let a bad message stop the master.
            self.logger.error("[POLL] failed to parse log message, skipping: %s", msg)
            return

        try:
            scanned = yaml.load(message, Loader=yaml.CLoader)
        except yaml.YAMLError:
            self.logger.error("[%s] data are not valid YAML, dropping", job_id)
            return

        # Look for "results" level
        try:
            message_lvl = scanned["lvl"]
            message_msg = scanned["msg"]
        except TypeError:
            self.logger.error("[%s] not a dictionary, dropping", job_id)
            return
        except KeyError:
            self.logger.error(
                "[%s] invalid log line, missing \"lvl\" or \"msg\" keys: %s",
                job_id, message)
            return

        # Find the handler (if available)
        if job_id not in self.jobs:
            # Query the database for the job
            try:
                job = TestJob.objects.get(id=job_id)
            except TestJob.DoesNotExist:
                self.logger.error("[%s] unknown job id", job_id)
                return

            self.logger.info("[%s] receiving logs from a new job", job_id)
            # Create the sub directories (if needed)
            mkdir(job.output_dir)
            self.jobs[job_id] = JobHandler(job)

        if message_lvl == "results":
            try:
                job = TestJob.objects.get(pk=job_id)
            except TestJob.DoesNotExist:
                self.logger.error("[%s] unknown job id", job_id)
                return
            meta_filename = create_metadata_store(message_msg, job)
            new_test_case = map_scanned_results(results=message_msg, job=job,
                                                meta_filename=meta_filename)
            if new_test_case is None:
                self.logger.warning(
                    "[%s] unable to map scanned results: %s",
                    job_id, message)
            else:
                self.test_cases.append(new_test_case)

            # Look for lava.job result
            if message_msg.get("definition") == "lava" and message_msg.get("case") == "job":
                # Flush cached test cases
                self.flush_test_cases()

                if message_msg.get("result") == "pass":
                    health = TestJob.HEALTH_COMPLETE
                    health_msg = "Complete"
                else:
                    health = TestJob.HEALTH_INCOMPLETE
                    health_msg = "Incomplete"
                self.logger.info("[%s] job status: %s", job_id, health_msg)

                infrastructure_error = (message_msg.get("error_type") == "Infrastructure")
                if infrastructure_error:
                    self.logger.info("[%s] Infrastructure error", job_id)

                # Update status.
                with transaction.atomic():
                    # TODO: find a way to lock actual_device
                    job = TestJob.objects.select_for_update() \
                                         .get(id=job_id)
                    job.go_state_finished(health, infrastructure_error)
                    job.save()

        # Mark the file handler as used
        self.jobs[job_id].last_usage = time.time()

        # n.b. logging here would produce a log entry for every message in every job.
        # The format is a list of dictionaries
        message = "- %s" % message

        # Write data
        self.jobs[job_id].write(message)

    def controler_socket(self):
        msg = self.controler.recv_multipart()
        try:
            master_id = u(msg[0])
            action = u(msg[1])
            ping_interval = int(msg[2])

            if master_id != "master":
                self.logger.error("Invalid master id '%s'. Should be 'master'",
                                  master_id)
                return
            if action != "PONG":
                self.logger.error("Invalid answer '%s'. Should be 'PONG'",
                                  action)
                return
        except (IndexError, ValueError):
            self.logger.error("Invalid message '%s'", msg)
            return

        if ping_interval < TIMEOUT:
            self.logger.error("invalid ping interval (%d) too small", ping_interval)
            return

        self.logger.debug("master => PONG(%d)", ping_interval)
        self.ping_interval = ping_interval
<commit_msg>Force device hc for ConfigurationError and LAVABug<commit_after># Copyright (C) 2017 Linaro Limited
#
# Author: Remi Duraffort <remi.duraffort@linaro.org>
#
# This file is part of LAVA Dispatcher.
#
# LAVA Dispatcher is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# LAVA Dispatcher is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along
# with this program; if not, see <http://www.gnu.org/licenses>.

# pylint: disable=wrong-import-order

import logging
import os
import time
import yaml
import zmq
import zmq.auth
from zmq.utils.strtypes import u
from zmq.auth.thread import ThreadAuthenticator

from django.db import connection, transaction
from django.db.utils import OperationalError, InterfaceError

from lava_results_app.models import TestCase
from lava_server.cmdutils import LAVADaemonCommand, watch_directory
from lava_scheduler_app.models import TestJob
from lava_scheduler_app.utils import mkdir
from lava_results_app.dbutils import map_scanned_results, create_metadata_store

# Constants
FORMAT = "%(asctime)-15s %(levelname)7s %(message)s"
TIMEOUT = 10
BULK_CREATE_TIMEOUT = 10
FD_TIMEOUT = 60

class JobHandler(object):  # pylint: disable=too-few-public-methods
    def __init__(self, job):
        self.output_dir = job.output_dir
        self.output = open(os.path.join(self.output_dir, 'output.yaml'), 'a+')
        self.last_usage = time.time()

    def write(self, message):
        self.output.write(message)
        self.output.write('\n')
        self.output.flush()

    def close(self):
        self.output.close()

class Command(LAVADaemonCommand):
    help = "LAVA log recorder"
    logger = None
    default_logfile = "/var/log/lava-server/lava-logs.log"

    def __init__(self, *args, **options):
        super(Command, self).__init__(*args, **options)
        self.logger = logging.getLogger("lava-logs")
        self.log_socket = None
        self.auth = None
        self.controler = None
        self.inotify_fd = None
        self.pipe_r = None
        self.poller = None
        self.cert_dir_path = None
        # List of logs
        self.jobs = {}
        # Keep test cases in memory
        self.test_cases = []
        # Master status
        self.last_ping = 0
        self.ping_interval = TIMEOUT

    def add_arguments(self, parser):
        super(Command, self).add_arguments(parser)

        net = parser.add_argument_group("network")
        net.add_argument('--socket',
                         default='tcp://*:5555',
                         help="Socket waiting for logs. Default: tcp://*:5555")
        net.add_argument('--master-socket',
                         default='tcp://localhost:5556',
                         help="Socket for master-slave communication. Default: tcp://localhost:5556")
        net.add_argument('--ipv6', default=False, action='store_true',
                         help="Enable IPv6 on the listening sockets")
        net.add_argument('--encrypt', default=False, action='store_true',
                         help="Encrypt messages")
        net.add_argument('--master-cert',
                         default='/etc/lava-dispatcher/certificates.d/master.key_secret',
                         help="Certificate for the master socket")
        net.add_argument('--slaves-certs',
                         default='/etc/lava-dispatcher/certificates.d',
                         help="Directory for slaves certificates")

    def handle(self, *args, **options):
        # Initialize logging.
        self.setup_logging("lava-logs", options["level"],
                           options["log_file"], FORMAT)

        self.logger.info("[INIT] Dropping privileges")
        if not self.drop_privileges(options['user'], options['group']):
            self.logger.error("[INIT] Unable to drop privileges")
            return

        # Create the sockets
        context = zmq.Context()
        self.log_socket = context.socket(zmq.PULL)
        self.controler = context.socket(zmq.ROUTER)
        self.controler.setsockopt(zmq.IDENTITY, b"lava-logs")
        # Limit the number of messages in the queue
        self.controler.setsockopt(zmq.SNDHWM, 2)
        # TODO: remove when Jessie is not supported
        if hasattr(zmq, "CONNECT_RID"):
            # From http://api.zeromq.org/4-2:zmq-setsockopt#toc5
            # "Immediately readies that connection for data transfer with the master"
            self.controler.setsockopt(zmq.CONNECT_RID, b"master")

        if options['ipv6']:
            self.logger.info("[INIT] Enabling IPv6")
            self.log_socket.setsockopt(zmq.IPV6, 1)
            self.controler.setsockopt(zmq.IPV6, 1)

        if options['encrypt']:
            self.logger.info("[INIT] Starting encryption")
            try:
                self.auth = ThreadAuthenticator(context)
                self.auth.start()
                self.logger.debug("[INIT] Opening master certificate: %s", options['master_cert'])
                master_public, master_secret = zmq.auth.load_certificate(options['master_cert'])
                self.logger.debug("[INIT] Using slaves certificates from: %s", options['slaves_certs'])
                self.auth.configure_curve(domain='*', location=options['slaves_certs'])
            except IOError as err:
                self.logger.error("[INIT] %s", err)
                self.auth.stop()
                return
            self.log_socket.curve_publickey = master_public
            self.log_socket.curve_secretkey = master_secret
            self.log_socket.curve_server = True
            self.controler.curve_publickey = master_public
            self.controler.curve_secretkey = master_secret
            self.controler.curve_serverkey = master_public

        self.logger.debug("[INIT] Watching %s", options["slaves_certs"])
        self.cert_dir_path = options["slaves_certs"]
        self.inotify_fd = watch_directory(options["slaves_certs"])
        if self.inotify_fd is None:
            self.logger.error("[INIT] Unable to start inotify")

        self.log_socket.bind(options['socket'])
        self.controler.connect(options['master_socket'])

        # Poll on the sockets. This allow to have a
        # nice timeout along with polling.
        self.poller = zmq.Poller()
        self.poller.register(self.log_socket, zmq.POLLIN)
        self.poller.register(self.controler, zmq.POLLIN)
        if self.inotify_fd is not None:
            self.poller.register(os.fdopen(self.inotify_fd), zmq.POLLIN)

        # Translate signals into zmq messages
        (self.pipe_r, _) = self.setup_zmq_signal_handler()
        self.poller.register(self.pipe_r, zmq.POLLIN)

        self.logger.info("[INIT] listening for logs")
        # PING right now: the master is waiting for this message to start
        # scheduling.
        self.controler.send_multipart([b"master", b"PING"])

        try:
            self.main_loop()
        except BaseException as exc:
            self.logger.error("[EXIT] Unknown exception raised, leaving!")
            self.logger.exception(exc)

        # Close the controler socket
        self.controler.close(linger=0)
        self.poller.unregister(self.controler)

        # Carefully close the logging socket as we don't want to lose messages
        self.logger.info("[EXIT] Disconnect logging socket and process messages")
        endpoint = u(self.log_socket.getsockopt(zmq.LAST_ENDPOINT))
        self.logger.debug("[EXIT] unbinding from '%s'", endpoint)
        self.log_socket.unbind(endpoint)

        # Empty the queue
        try:
            while self.wait_for_messages(True):
                # Flush test cases cache for every iteration because we might
                # get killed soon.
                self.flush_test_cases()
        except BaseException as exc:
            self.logger.error("[EXIT] Unknown exception raised, leaving!")
            self.logger.exception(exc)
        finally:
            # Last flush
            self.flush_test_cases()
            self.logger.info("[EXIT] Closing the logging socket: the queue is empty")
            self.log_socket.close()
            if options['encrypt']:
                self.auth.stop()
            context.term()

    def flush_test_cases(self):
        if self.test_cases:
            self.logger.info("Saving %d test cases", len(self.test_cases))
            TestCase.objects.bulk_create(self.test_cases)
            self.test_cases = []

    def main_loop(self):
        last_gc = time.time()
        last_bulk_create = time.time()

        # Wait for messages
        # TODO: fix timeout computation
        while self.wait_for_messages(False):
            now = time.time()

            # Dump TestCase into the database
            if now - last_bulk_create > BULK_CREATE_TIMEOUT:
                last_bulk_create = now
                self.flush_test_cases()

            # Close old file handlers
            if now - last_gc > FD_TIMEOUT:
                last_gc = now
                # Iterate while removing keys is not compatible with iterator
                for job_id in list(self.jobs.keys()):  # pylint: disable=consider-iterating-dictionary
                    if now - self.jobs[job_id].last_usage > FD_TIMEOUT:
                        self.logger.info("[%s] closing log file", job_id)
                        self.jobs[job_id].close()
                        del self.jobs[job_id]

            # Ping the master
            if now - self.last_ping > self.ping_interval:
                self.logger.debug("PING => master")
                self.last_ping = now
                self.controler.send_multipart([b"master", b"PING"])

    def wait_for_messages(self, leaving):
        try:
            try:
                sockets = dict(self.poller.poll(TIMEOUT * 1000))
            except zmq.error.ZMQError as exc:
                self.logger.error("[POLL] zmq error: %s", str(exc))
                return True

            # Messages
            if sockets.get(self.log_socket) == zmq.POLLIN:
                self.logging_socket()
                return True

            # Signals
            elif sockets.get(self.pipe_r) == zmq.POLLIN:
                # remove the message from the queue
                os.read(self.pipe_r, 1)

                if not leaving:
                    self.logger.info("[POLL] received a signal, leaving")
                    return False
                else:
                    self.logger.warning("[POLL] signal already handled, please wait for the process to exit")
                    return True

            # Pong received
            elif sockets.get(self.controler) == zmq.POLLIN:
                self.controler_socket()
                return True

            # Inotify socket
            if sockets.get(self.inotify_fd) == zmq.POLLIN:
                os.read(self.inotify_fd, 4096)
                self.logger.debug("[AUTH] Reloading certificates from %s",
                                  self.cert_dir_path)
                self.auth.configure_curve(domain='*',
                                          location=self.cert_dir_path)

            # Nothing received
            else:
                return not leaving

        except (OperationalError, InterfaceError):
            self.logger.info("[RESET] database connection reset")
            connection.close()
        return True

    def logging_socket(self):
        msg = self.log_socket.recv_multipart()
        try:
            (job_id, message) = (u(m) for m in msg)  # pylint: disable=unbalanced-tuple-unpacking
        except ValueError:
            # do not let a bad message stop the master.
            self.logger.error("[POLL] failed to parse log message, skipping: %s", msg)
            return

        try:
            scanned = yaml.load(message, Loader=yaml.CLoader)
        except yaml.YAMLError:
            self.logger.error("[%s] data are not valid YAML, dropping", job_id)
            return

        # Look for "results" level
        try:
            message_lvl = scanned["lvl"]
            message_msg = scanned["msg"]
        except TypeError:
            self.logger.error("[%s] not a dictionary, dropping", job_id)
            return
        except KeyError:
            self.logger.error(
                "[%s] invalid log line, missing \"lvl\" or \"msg\" keys: %s",
                job_id, message)
            return

        # Find the handler (if available)
        if job_id not in self.jobs:
            # Query the database for the job
            try:
                job = TestJob.objects.get(id=job_id)
            except TestJob.DoesNotExist:
                self.logger.error("[%s] unknown job id", job_id)
                return

            self.logger.info("[%s] receiving logs from a new job", job_id)
            # Create the sub directories (if needed)
            mkdir(job.output_dir)
            self.jobs[job_id] = JobHandler(job)

        if message_lvl == "results":
            try:
                job = TestJob.objects.get(pk=job_id)
            except TestJob.DoesNotExist:
                self.logger.error("[%s] unknown job id", job_id)
                return
            meta_filename = create_metadata_store(message_msg, job)
            new_test_case = map_scanned_results(results=message_msg, job=job,
                                                meta_filename=meta_filename)
            if new_test_case is None:
                self.logger.warning(
                    "[%s] unable to map scanned results: %s",
                    job_id, message)
            else:
                self.test_cases.append(new_test_case)

            # Look for lava.job result
            if message_msg.get("definition") == "lava" and message_msg.get("case") == "job":
                # Flush cached test cases
                self.flush_test_cases()

                if message_msg.get("result") == "pass":
                    health = TestJob.HEALTH_COMPLETE
                    health_msg = "Complete"
                else:
                    health = TestJob.HEALTH_INCOMPLETE
                    health_msg = "Incomplete"
                self.logger.info("[%s] job status: %s", job_id, health_msg)

                infrastructure_error = (message_msg.get("error_type") in ["Bug",
                                                                          "Configuration",
                                                                          "Infrastructure"])
                if infrastructure_error:
                    self.logger.info("[%s] Infrastructure error", job_id)

                # Update status.
                with transaction.atomic():
                    # TODO: find a way to lock actual_device
                    job = TestJob.objects.select_for_update() \
                                         .get(id=job_id)
                    job.go_state_finished(health, infrastructure_error)
                    job.save()

        # Mark the file handler as used
        self.jobs[job_id].last_usage = time.time()

        # n.b. logging here would produce a log entry for every message in every job.
        # The format is a list of dictionaries
        message = "- %s" % message

        # Write data
        self.jobs[job_id].write(message)

    def controler_socket(self):
        msg = self.controler.recv_multipart()
        try:
            master_id = u(msg[0])
            action = u(msg[1])
            ping_interval = int(msg[2])

            if master_id != "master":
                self.logger.error("Invalid master id '%s'. Should be 'master'",
                                  master_id)
                return
            if action != "PONG":
                self.logger.error("Invalid answer '%s'. Should be 'PONG'",
                                  action)
                return
        except (IndexError, ValueError):
            self.logger.error("Invalid message '%s'", msg)
            return

        if ping_interval < TIMEOUT:
            self.logger.error("invalid ping interval (%d) too small", ping_interval)
            return

        self.logger.debug("master => PONG(%d)", ping_interval)
        self.ping_interval = ping_interval
<|endoftext|>


<commit_before>"""
    Generic base class for API calls using forms for input validation

    Returns a JsonResponse with varying status codes for error handling.

    TODO user group authentication support
    TODO sensitive parameter support
"""

from django.http import JsonResponse, HttpRequest
from django.core.exceptions import ValidationError
from django.views.generic import View
from django.utils.translation import ugettext as _
from django.conf import settings
from django import forms
from hackfsu_com.util import acl
from hackfsu_com.util.exceptions import InternalServerError, ExternalUserError
import logging

def format_request_info(request: HttpRequest, input_data: dict, req, res):
    return '\tRequest: {}\n\tRaw Input: {}\n\tReq: {}\n\tRes: {}'.format(
        str(request), str(input_data), str(req), str(res)
    )

class ApiView(View):
    http_method_names = ['post']        # Override to allow GET
    request_form_class = forms.Form     # Override each time
    response_form_class = forms.Form    # Override each time
    access_manager = acl.AccessManager()

    def get(self, request: HttpRequest):
        return self.process(request, request.GET)

    def post(self, request: HttpRequest):
        return self.process(request, request.POST)

    def process(self, request: HttpRequest, input_data: dict):
        """ Validates input and attempts to preform work() logic. Returns the correct JsonResponse """

        # Authenticate Access
        if not self.authenticate(request):
            return JsonResponse({
                'cause': _('Unauthorized')
            }, status=401)

        req = None
        res = None

        # Preform request
        try:
            # Validate & clean request
            request_form = self.request_form_class(input_data, request.FILES)
            if not request_form.is_valid():
                raise ExternalUserError(ValidationError(request_form.errors.as_data()))
            req = request_form.cleaned_data
            res = {}

            # Preform desired api task and populate response (res) object
            try:
                self.work(request, req, res)
            except ValidationError as e:
                raise ExternalUserError(e)

            # Validate response
            response_form = self.response_form_class(res)
            if not response_form.is_valid():
                raise InternalServerError(ValidationError(response_form.errors.as_data()))
            res = response_form.cleaned_data

            # Successful api call!
            return JsonResponse(res)

        except ExternalUserError as error:
            if settings.DEBUG:
                logging.error(format_request_info(request, input_data, res, req))
                error.log()
            return error.json_response()
        except InternalServerError as error:
            request_info = format_request_info(request, input_data, res, req)
            logging.error(request_info)
            error.log()

            if not settings.DEBUG:
                error.email_log_to_dev(request_info=request_info, user=request.user)

            return error.json_response(include_message=False)
        except Exception as e:
            error = InternalServerError(e)
            request_info = format_request_info(request, input_data, res, req)
            logging.error(request_info)
            error.log()

            if not settings.DEBUG:
                error.email_log_to_dev(request_info=request_info, user=request.user)

            return error.json_response(include_message=False)

    def work(self, request: HttpRequest, req: dict, res: dict):
        """
            Preforms api request logic
            :param request Django request object (only use this if necessary)
            :param req cleaned and valid request data
            :param res response data to be checked after this call
        """
        pass

    def authenticate(self, request):
        """ To be overridden if necessary. Should still be called with super """
        return self.access_manager.check_user(request.user)
<commit_msg>kwarks for ApiView<commit_after>"""
    Generic base class for API calls using forms for input validation

    Returns a JsonResponse with varying status codes for error handling.

    TODO user group authentication support
    TODO sensitive parameter support
"""

from django.http import JsonResponse, HttpRequest
from django.core.exceptions import ValidationError
from django.views.generic import View
from django.utils.translation import ugettext as _
from django.conf import settings
from django import forms
from hackfsu_com.util import acl
from hackfsu_com.util.exceptions import InternalServerError, ExternalUserError
import logging

def format_request_info(request: HttpRequest, input_data: dict, req, res):
    return '\tRequest: {}\n\tRaw Input: {}\n\tReq: {}\n\tRes: {}'.format(
        str(request), str(input_data), str(req), str(res)
    )

class ApiView(View):
    http_method_names = ['post']        # Override to allow GET
    request_form_class = forms.Form     # Override each time
    response_form_class = forms.Form    # Override each time
    access_manager = acl.AccessManager()

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.kwargs = list()
        self.args = list()

    def get(self, request: HttpRequest,  *args, **kwargs):
        self.args = args
        self.kwargs = kwargs
        return self.process(request, request.GET)

    def post(self, request: HttpRequest, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs
        return self.process(request, request.POST)

    def process(self, request: HttpRequest, input_data: dict):
        """ Validates input and attempts to preform work() logic. Returns the correct JsonResponse """

        # Authenticate Access
        if not self.authenticate(request):
            return JsonResponse({
                'cause': _('Unauthorized')
            }, status=401)

        req = None
        res = None

        # Preform request
        try:
            # Validate & clean request
            request_form = self.request_form_class(input_data, request.FILES)
            if not request_form.is_valid():
                raise ExternalUserError(ValidationError(request_form.errors.as_data()))
            req = request_form.cleaned_data
            res = {}

            # Preform desired api task and populate response (res) object
            try:
                self.work(request, req, res)
            except ValidationError as e:
                raise ExternalUserError(e)

            # Validate response
            response_form = self.response_form_class(res)
            if not response_form.is_valid():
                raise InternalServerError(ValidationError(response_form.errors.as_data()))
            res = response_form.cleaned_data

            # Successful api call!
            return JsonResponse(res)

        except ExternalUserError as error:
            if settings.DEBUG:
                logging.error(format_request_info(request, input_data, res, req))
                error.log()
            return error.json_response()
        except InternalServerError as error:
            request_info = format_request_info(request, input_data, res, req)
            logging.error(request_info)
            error.log()

            if not settings.DEBUG:
                error.email_log_to_dev(request_info=request_info, user=request.user)

            return error.json_response(include_message=False)
        except Exception as e:
            error = InternalServerError(e)
            request_info = format_request_info(request, input_data, res, req)
            logging.error(request_info)
            error.log()

            if not settings.DEBUG:
                error.email_log_to_dev(request_info=request_info, user=request.user)

            return error.json_response(include_message=False)

    def work(self, request: HttpRequest, req: dict, res: dict):
        """
            Preforms api request logic
            :param request Django request object (only use this if necessary)
            :param req cleaned and valid request data
            :param res response data to be checked after this call
        """
        pass

    def authenticate(self, request):
        """ To be overridden if necessary. Should still be called with super """
        return self.access_manager.check_user(request.user)
<|endoftext|>


<commit_before># future
from __future__ import annotations

# stdlib
from collections.abc import Sequence
from typing import Any
from typing import Callable
from typing import Dict
from typing import List
from typing import Optional
from typing import Tuple
from typing import Union

# third party
from nacl.signing import VerifyKey
import numpy as np

# relative
from ....core.adp.data_subject_ledger import DataSubjectLedger
from ....core.adp.data_subject_list import DataSubjectList
from ....core.adp.data_subject_list import liststrtonumpyutf8
from ....core.adp.data_subject_list import numpyutf8tolist
from ....core.adp.entity import Entity
from ....lib.numpy.array import capnp_deserialize
from ....lib.numpy.array import capnp_serialize
from ...common.serde.capnp import CapnpModule
from ...common.serde.capnp import get_capnp_schema
from ...common.serde.capnp import serde_magic_header
from ...common.serde.serializable import serializable
from ...common.uid import UID
from ...pointer.pointer import Pointer
from ..ancestors import AutogradTensorAncestor
from ..lazy_repeat_array import lazyrepeatarray
from ..passthrough import AcceptableSimpleType  # type: ignore
from ..passthrough import PassthroughTensor  # type: ignore
from ..passthrough import SupportedChainType  # type: ignore
from ..passthrough import is_acceptable_simple_type  # type: ignore
from .adp_tensor import ADPTensor
from .gamma_tensor import GammaTensor
from .initial_gamma import InitialGammaTensor
from .initial_gamma import IntermediateGammaTensor
from .single_entity_phi import SingleEntityPhiTensor

@serializable(recursive_serde=True)
class NDimEntityPhiTensorPointer(Pointer):
    __name__ = "NDimEntityPhiTensorPointer"
    __module__ = "syft.core.tensor.autodp.ndim_entity_phi"
    __attr_allowlist__ = [
        # default pointer attrs
        "points_to_object_with_path",
        "pointer_name",
        "id_at_location",
        "location",
        "tags",
        "description",
        "object_type",
        "attribute_name",
        "public_shape",
        "_exhausted",
        "gc_enabled",
        "is_enum",
        # ndim attrs
        "entities",
        "min_vals",
        "max_vals",
        "client",
        "public_dtype",
    ]

    # TODO :should create serialization for Entity List

    def __init__(
        self,
        entities: DataSubjectList,
        min_vals: np.typing.ArrayLike,
        max_vals: np.typing.ArrayLike,
        client: Any,
        id_at_location: Optional[UID] = None,
        object_type: str = "",
        tags: Optional[List[str]] = None,
        description: str = "",
        public_shape: Optional[Tuple[int, ...]] = None,
        public_dtype: Optional[np.dtype] = None,
    ):
        super().__init__(
            client=client,
            id_at_location=id_at_location,
            object_type=object_type,
            tags=tags,
            description=description,
        )

        self.min_vals = min_vals
        self.max_vals = max_vals
        self.entities = entities
        self.public_shape = public_shape
        self.public_dtype = public_dtype

@serializable(capnp_bytes=True)
class NDimEntityPhiTensor(PassthroughTensor, AutogradTensorAncestor, ADPTensor):
    PointerClassOverride = NDimEntityPhiTensorPointer
    # __attr_allowlist__ = ["child", "min_vals", "max_vals", "entities"]
    __slots__ = (
        "child",
        "min_vals",
        "max_vals",
        "entities",
    )

    def __init__(
        self,
        child: Sequence,
        entities: Union[List[Entity], DataSubjectList],
        min_vals: np.ndarray,
        max_vals: np.ndarray,
        row_type: SingleEntityPhiTensor = SingleEntityPhiTensor,  # type: ignore
    ) -> None:

        # child = the actual private data
        super().__init__(child)

        # lazyrepeatarray matching the shape of child
        if not isinstance(min_vals, lazyrepeatarray):
            min_vals = lazyrepeatarray(data=min_vals, shape=child.shape)  # type: ignore
        if not isinstance(max_vals, lazyrepeatarray):
            max_vals = lazyrepeatarray(data=max_vals, shape=child.shape)  # type: ignore
        self.min_vals = min_vals
        self.max_vals = max_vals

        if not isinstance(entities, DataSubjectList):
            entities = DataSubjectList.from_objs(entities)

        self.entities = entities

    @property
    def proxy_public_kwargs(self) -> Dict[str, Any]:
        return {
            "min_vals": self.min_vals,
            "max_vals": self.max_vals,
            "entities": self.entities,
        }

    @staticmethod
    def from_rows(rows: Sequence) -> NDimEntityPhiTensor:
        if len(rows) < 1 or not isinstance(rows[0], SingleEntityPhiTensor):
            raise Exception(
                "NDimEntityPhiTensor.from_rows requires a list of SingleEntityPhiTensors"
            )

        # create lazyrepeatarrays of the first element
        first_row = rows[0]
        min_vals = lazyrepeatarray(
            data=first_row.min_vals,
            shape=tuple([len(rows)] + list(first_row.min_vals.shape)),
        )
        max_vals = lazyrepeatarray(
            data=first_row.max_vals,
            shape=tuple([len(rows)] + list(first_row.max_vals.shape)),
        )

        # collect entities and children into numpy arrays
        entity_list = []
        child_list = []
        for row in rows:
            entity_list.append(row.entity)
            child_list.append(row.child)
        entities = DataSubjectList.from_objs(entities=entity_list)
        child = np.stack(child_list)

        # use new constructor
        return NDimEntityPhiTensor(
            child=child,
            min_vals=min_vals,
            max_vals=max_vals,
            entities=entities,
        )

    def init_pointer(
        self,
        client: Any,
        id_at_location: Optional[UID] = None,
        object_type: str = "",
        tags: Optional[List[str]] = None,
        description: str = "",
    ) -> NDimEntityPhiTensorPointer:
        return NDimEntityPhiTensorPointer(
            # Arguments specifically for SEPhiTensor
            entities=self.entities,
            min_vals=self.min_vals,
            max_vals=self.max_vals,
            # Arguments required for a Pointer to work
            client=client,
            id_at_location=id_at_location,
            object_type=object_type,
            tags=tags,
            description=description,
        )

    @property
    def gamma(self) -> GammaTensor:
        """Property to cast this tensor into a GammaTensor"""
        return self.create_gamma()

    def copy(self, order: Optional[str] = "K") -> NDimEntityPhiTensor:
        """Return copy of the given object"""

        return NDimEntityPhiTensor(
            child=self.child.copy(order=order),
            min_vals=self.min_vals.copy(order=order),
            max_vals=self.max_vals.copy(order=order),
            entities=self.entities.copy(order=order),
        )

    def all(self) -> bool:
        return self.child.all()

    def any(self) -> bool:
        return self.child.any()

    def copy_with(self, child: np.ndarray) -> NDimEntityPhiTensor:
        new_tensor = self.copy()
        new_tensor.child = child
        return new_tensor

    def create_gamma(self) -> GammaTensor:
        """Return a new Gamma tensor based on this phi tensor"""

        # if scalar_manager is None:
        #     scalar_manager = self.scalar_manager

        # Gamma expects an entity for each scalar
        # entities = np.array([self.entity] * np.array(self.child.shape).prod()).reshape(
        #     self.shape
        # )

        # TODO: update InitialGammaTensor to handle DataSubjectList
        # TODO: check if values needs to be a JAX array or if numpy will suffice

        return GammaTensor(
            value=self.child,
            data_subjects=self.entities,
            min_val=self.min_vals,
            max_val=self.max_vals,
        )

    def publish(
        self,
        get_budget_for_user: Callable,
        deduct_epsilon_for_user: Callable,
        ledger: DataSubjectLedger,
        sigma: float,
        user_key: VerifyKey,
    ) -> AcceptableSimpleType:
        print("PUBLISHING TO GAMMA:")
        print(self.child)
        return self.gamma.publish(
            get_budget_for_user=get_budget_for_user,
            deduct_epsilon_for_user=deduct_epsilon_for_user,
            ledger=ledger,
            sigma=sigma,
        )

    @property
    def value(self) -> np.ndarray:
        return self.child

    def astype(self, np_type: np.dtype) -> NDimEntityPhiTensor:
        return self.__class__(
            child=self.child.astype(np_type),
            entities=self.entities,
            min_vals=self.min_vals.astype(np_type),
            max_vals=self.max_vals.astype(np_type),
            # scalar_manager=self.scalar_manager,
        )

    @property
    def shape(self) -> Tuple[Any, ...]:
        return self.child.shape

    def __repr__(self) -> str:
        """Pretty print some information, optimized for Jupyter notebook viewing."""
        return (
            f"{self.__class__.__name__}(child={self.child.shape}, "
            + f"min_vals={self.min_vals}, max_vals={self.max_vals})"
        )

    def __eq__(  # type: ignore
        self, other: Any
    ) -> Union[NDimEntityPhiTensor, IntermediateGammaTensor, GammaTensor]:
        # TODO: what about entities and min / max values?
        if is_acceptable_simple_type(other) or len(self.child) == len(other.child):
            gamma_output = False
            if is_acceptable_simple_type(other):
                result = self.child == other
            else:
                # check entities match, if they dont gamma_output = True
                #
                result = self.child == other.child
                if isinstance(result, InitialGammaTensor):
                    gamma_output = True
            if not gamma_output:
                # min_vals=self.min_vals * 0.0,
                # max_vals=self.max_vals * 0.0 + 1.0,
                return self.copy_with(child=result)
            else:
                return self.copy_with(child=result).gamma
        else:
            raise Exception(
                "Tensor dims do not match for __eq__: "
                + f"{len(self.child)} != {len(other.child)}"
            )

    def __add__(
        self, other: SupportedChainType
    ) -> Union[NDimEntityPhiTensor, IntermediateGammaTensor, GammaTensor]:

        # if the tensor being added is also private
        if isinstance(other, NDimEntityPhiTensor):
            if self.entities != other.entities:
                return self.gamma + other.gamma

            return NDimEntityPhiTensor(
                child=self.child + other.child,
                min_vals=self.min_vals + other.min_vals,
                max_vals=self.max_vals + other.max_vals,
                entities=self.entities,
                # scalar_manager=self.scalar_manager,
            )

        # if the tensor being added is a public tensor / int / float / etc.
        elif is_acceptable_simple_type(other):
            return NDimEntityPhiTensor(
                child=self.child + other,
                min_vals=self.min_vals + other,
                max_vals=self.max_vals + other,
                entities=self.entities,
                # scalar_manager=self.scalar_manager,
            )

        elif isinstance(other, IntermediateGammaTensor):
            return self.gamma + other
        else:
            print("Type is unsupported:" + str(type(other)))
            raise NotImplementedError

    def sum(
        self, axis: Optional[Union[int, Tuple[int, ...]]] = None
    ) -> Union[NDimEntityPhiTensor, GammaTensor]:
        # TODO: Add support for axes arguments later
        if len(self.entities.one_hot_lookup) == 1:
            return NDimEntityPhiTensor(
                child=self.child.sum(),
                min_vals=self.min_vals.sum(axis=None),
                max_vals=self.max_vals.sum(axis=None),
                entities=DataSubjectList.from_objs(
                    self.entities.one_hot_lookup[0]
                ),  # Need to check this
            )
        return GammaTensor(
            value=np.array(self.child.sum()),
            data_subjects=self.entities.sum(),
            min_val=float(self.min_vals.sum(axis=None)),
            max_val=float(self.max_vals.sum(axis=None)),
            inputs=self.child,
        )

    def __ne__(  # type: ignore
        self, other: Any
    ) -> Union[NDimEntityPhiTensor, IntermediateGammaTensor, GammaTensor]:
        # TODO: what about entities and min / max values?
        if is_acceptable_simple_type(other) or len(self.child) == len(other.child):
            gamma_output = False
            if is_acceptable_simple_type(other):
                result = self.child != other
            else:
                # check entities match, if they dont gamma_output = True
                #
                result = self.child != other.child
                if isinstance(result, InitialGammaTensor):
                    gamma_output = True
            if not gamma_output:
                return self.copy_with(child=result)
            else:
                return self.copy_with(child=result).gamma
        else:
            raise Exception(
                "Tensor dims do not match for __eq__: "
                + f"{len(self.child)} != {len(other.child)}"
            )

    def __neg__(self) -> NDimEntityPhiTensor:

        return NDimEntityPhiTensor(
            child=self.child * -1,
            min_vals=self.max_vals * -1,
            max_vals=self.min_vals * -1,
            entities=self.entities,
        )

    def _object2bytes(self) -> bytes:
        schema = get_capnp_schema(schema_file="ndept.capnp")

        ndept_struct: CapnpModule = schema.NDEPT  # type: ignore
        ndept_msg = ndept_struct.new_message()
        # this is how we dispatch correct deserialization of bytes
        ndept_msg.magicHeader = serde_magic_header(type(self))

        ndept_msg.child = capnp_serialize(self.child)
        ndept_msg.minVals = capnp_serialize(self.min_vals.data)
        ndept_msg.maxVals = capnp_serialize(self.max_vals.data)
        ndept_msg.dataSubjectsIndexed = capnp_serialize(
            self.entities.data_subjects_indexed
        )

        ndept_msg.oneHotLookup = capnp_serialize(
            liststrtonumpyutf8(self.entities.one_hot_lookup)
        )

        # to pack or not to pack?
        # to_bytes = ndept_msg.to_bytes()

        return ndept_msg.to_bytes_packed()

    @staticmethod
    def _bytes2object(buf: bytes) -> NDimEntityPhiTensor:
        schema = get_capnp_schema(schema_file="ndept.capnp")
        ndept_struct: CapnpModule = schema.NDEPT  # type: ignore
        # https://stackoverflow.com/questions/48458839/capnproto-maximum-filesize
        MAX_TRAVERSAL_LIMIT = 2**64 - 1
        # to pack or not to pack?
        # ndept_msg = ndept_struct.from_bytes(buf, traversal_limit_in_words=2 ** 64 - 1)
        ndept_msg = ndept_struct.from_bytes_packed(
            buf, traversal_limit_in_words=MAX_TRAVERSAL_LIMIT
        )

        child = capnp_deserialize(ndept_msg.child)
        min_vals = capnp_deserialize(ndept_msg.minVals)
        max_vals = capnp_deserialize(ndept_msg.maxVals)
        data_subjects_indexed = capnp_deserialize(ndept_msg.dataSubjectsIndexed)
        one_hot_lookup = numpyutf8tolist(capnp_deserialize(ndept_msg.oneHotLookup))

        entity_list = DataSubjectList(one_hot_lookup, data_subjects_indexed)

        return NDimEntityPhiTensor(
            child=child, min_vals=min_vals, max_vals=max_vals, entities=entity_list
        )
<commit_msg>empty commit<commit_after># future
from __future__ import annotations

# stdlib
from collections.abc import Sequence
from typing import Any
from typing import Callable
from typing import Dict
from typing import List
from typing import Optional
from typing import Tuple
from typing import Union

# third party
from nacl.signing import VerifyKey
import numpy as np

# relative
from ....core.adp.data_subject_ledger import DataSubjectLedger
from ....core.adp.data_subject_list import DataSubjectList
from ....core.adp.data_subject_list import liststrtonumpyutf8
from ....core.adp.data_subject_list import numpyutf8tolist
from ....core.adp.entity import Entity
from ....lib.numpy.array import capnp_deserialize
from ....lib.numpy.array import capnp_serialize
from ...common.serde.capnp import CapnpModule
from ...common.serde.capnp import get_capnp_schema
from ...common.serde.capnp import serde_magic_header
from ...common.serde.serializable import serializable
from ...common.uid import UID
from ...pointer.pointer import Pointer
from ..ancestors import AutogradTensorAncestor
from ..lazy_repeat_array import lazyrepeatarray
from ..passthrough import AcceptableSimpleType  # type: ignore
from ..passthrough import PassthroughTensor  # type: ignore
from ..passthrough import SupportedChainType  # type: ignore
from ..passthrough import is_acceptable_simple_type  # type: ignore
from .adp_tensor import ADPTensor
from .gamma_tensor import GammaTensor
from .initial_gamma import InitialGammaTensor
from .initial_gamma import IntermediateGammaTensor
from .single_entity_phi import SingleEntityPhiTensor

@serializable(recursive_serde=True)
class NDimEntityPhiTensorPointer(Pointer):
    __name__ = "NDimEntityPhiTensorPointer"
    __module__ = "syft.core.tensor.autodp.ndim_entity_phi"
    __attr_allowlist__ = [
        # default pointer attrs
        "points_to_object_with_path",
        "pointer_name",
        "id_at_location",
        "location",
        "tags",
        "description",
        "object_type",
        "attribute_name",
        "public_shape",
        "_exhausted",
        "gc_enabled",
        "is_enum",
        # ndim attrs
        "entities",
        "min_vals",
        "max_vals",
        "client",
        "public_dtype",
    ]

    # TODO :should create serialization for Entity List

    def __init__(
        self,
        entities: DataSubjectList,
        min_vals: np.typing.ArrayLike,
        max_vals: np.typing.ArrayLike,
        client: Any,
        id_at_location: Optional[UID] = None,
        object_type: str = "",
        tags: Optional[List[str]] = None,
        description: str = "",
        public_shape: Optional[Tuple[int, ...]] = None,
        public_dtype: Optional[np.dtype] = None,
    ):
        super().__init__(
            client=client,
            id_at_location=id_at_location,
            object_type=object_type,
            tags=tags,
            description=description,
        )

        self.min_vals = min_vals
        self.max_vals = max_vals
        self.entities = entities
        self.public_shape = public_shape
        self.public_dtype = public_dtype

@serializable(capnp_bytes=True)
class NDimEntityPhiTensor(PassthroughTensor, AutogradTensorAncestor, ADPTensor):
    PointerClassOverride = NDimEntityPhiTensorPointer
    # __attr_allowlist__ = ["child", "min_vals", "max_vals", "entities"]
    __slots__ = (
        "child",
        "min_vals",
        "max_vals",
        "entities",
    )

    def __init__(
        self,
        child: Sequence,
        entities: Union[List[Entity], DataSubjectList],
        min_vals: np.ndarray,
        max_vals: np.ndarray,
        row_type: SingleEntityPhiTensor = SingleEntityPhiTensor,  # type: ignore
    ) -> None:

        # child = the actual private data
        super().__init__(child)

        # lazyrepeatarray matching the shape of child
        if not isinstance(min_vals, lazyrepeatarray):
            min_vals = lazyrepeatarray(data=min_vals, shape=child.shape)  # type: ignore
        if not isinstance(max_vals, lazyrepeatarray):
            max_vals = lazyrepeatarray(data=max_vals, shape=child.shape)  # type: ignore
        self.min_vals = min_vals
        self.max_vals = max_vals

        if not isinstance(entities, DataSubjectList):
            entities = DataSubjectList.from_objs(entities)

        self.entities = entities

    @property
    def proxy_public_kwargs(self) -> Dict[str, Any]:
        return {
            "min_vals": self.min_vals,
            "max_vals": self.max_vals,
            "entities": self.entities,
        }

    @staticmethod
    def from_rows(rows: Sequence) -> NDimEntityPhiTensor:
        if len(rows) < 1 or not isinstance(rows[0], SingleEntityPhiTensor):
            raise Exception(
                "NDimEntityPhiTensor.from_rows requires a list of SingleEntityPhiTensors"
            )

        # create lazyrepeatarrays of the first element
        first_row = rows[0]
        min_vals = lazyrepeatarray(
            data=first_row.min_vals,
            shape=tuple([len(rows)] + list(first_row.min_vals.shape)),
        )
        max_vals = lazyrepeatarray(
            data=first_row.max_vals,
            shape=tuple([len(rows)] + list(first_row.max_vals.shape)),
        )

        # collect entities and children into numpy arrays
        entity_list = []
        child_list = []
        for row in rows:
            entity_list.append(row.entity)
            child_list.append(row.child)
        entities = DataSubjectList.from_objs(entities=entity_list)
        child = np.stack(child_list)

        # use new constructor
        return NDimEntityPhiTensor(
            child=child,
            min_vals=min_vals,
            max_vals=max_vals,
            entities=entities,
        )

    def init_pointer(
        self,
        client: Any,
        id_at_location: Optional[UID] = None,
        object_type: str = "",
        tags: Optional[List[str]] = None,
        description: str = "",
    ) -> NDimEntityPhiTensorPointer:
        return NDimEntityPhiTensorPointer(
            # Arguments specifically for SEPhiTensor
            entities=self.entities,
            min_vals=self.min_vals,
            max_vals=self.max_vals,
            # Arguments required for a Pointer to work
            client=client,
            id_at_location=id_at_location,
            object_type=object_type,
            tags=tags,
            description=description,
        )

    @property
    def gamma(self) -> GammaTensor:
        """Property to cast this tensor into a GammaTensor"""
        return self.create_gamma()

    def copy(self, order: Optional[str] = "K") -> NDimEntityPhiTensor:
        """Return copy of the given object"""

        return NDimEntityPhiTensor(
            child=self.child.copy(order=order),
            min_vals=self.min_vals.copy(order=order),
            max_vals=self.max_vals.copy(order=order),
            entities=self.entities.copy(order=order),
        )

    def all(self) -> bool:
        return self.child.all()

    def any(self) -> bool:
        return self.child.any()

    def copy_with(self, child: np.ndarray) -> NDimEntityPhiTensor:
        new_tensor = self.copy()
        new_tensor.child = child
        return new_tensor

    def create_gamma(self) -> GammaTensor:
        """Return a new Gamma tensor based on this phi tensor"""

        # if scalar_manager is None:
        #     scalar_manager = self.scalar_manager

        # Gamma expects an entity for each scalar
        # entities = np.array([self.entity] * np.array(self.child.shape).prod()).reshape(
        #     self.shape
        # )

        # TODO: update InitialGammaTensor to handle DataSubjectList
        # TODO: check if values needs to be a JAX array or if numpy will suffice

        return GammaTensor(
            value=self.child,
            data_subjects=self.entities,
            min_val=self.min_vals,
            max_val=self.max_vals,
        )

    def publish(
        self,
        get_budget_for_user: Callable,
        deduct_epsilon_for_user: Callable,
        ledger: DataSubjectLedger,
        sigma: float,
        user_key: VerifyKey,
    ) -> AcceptableSimpleType:
        print("PUBLISHING TO GAMMA:")
        print(self.child)
        return self.gamma.publish(
            get_budget_for_user=get_budget_for_user,
            deduct_epsilon_for_user=deduct_epsilon_for_user,
            ledger=ledger,
            sigma=sigma,
        )

    @property
    def value(self) -> np.ndarray:
        return self.child

    def astype(self, np_type: np.dtype) -> NDimEntityPhiTensor:
        return self.__class__(
            child=self.child.astype(np_type),
            entities=self.entities,
            min_vals=self.min_vals.astype(np_type),
            max_vals=self.max_vals.astype(np_type),
            # scalar_manager=self.scalar_manager,
        )

    @property
    def shape(self) -> Tuple[Any, ...]:
        return self.child.shape

    def __repr__(self) -> str:
        """Pretty print some information, optimized for Jupyter notebook viewing."""
        return (
            f"{self.__class__.__name__}(child={self.child.shape}, "
            + f"min_vals={self.min_vals}, max_vals={self.max_vals})"
        )

    def __eq__(  # type: ignore
        self, other: Any
    ) -> Union[NDimEntityPhiTensor, IntermediateGammaTensor, GammaTensor]:
        # TODO: what about entities and min / max values?
        if is_acceptable_simple_type(other) or len(self.child) == len(other.child):
            gamma_output = False
            if is_acceptable_simple_type(other):
                result = self.child == other
            else:
                # check entities match, if they dont gamma_output = True
                #
                result = self.child == other.child
                if isinstance(result, InitialGammaTensor):
                    gamma_output = True
            if not gamma_output:
                # min_vals=self.min_vals * 0.0,
                # max_vals=self.max_vals * 0.0 + 1.0,
                return self.copy_with(child=result)
            else:
                return self.copy_with(child=result).gamma
        else:
            raise Exception(
                "Tensor dims do not match for __eq__: "
                + f"{len(self.child)} != {len(other.child)}"
            )

    def __add__(
        self, other: SupportedChainType
    ) -> Union[NDimEntityPhiTensor, IntermediateGammaTensor, GammaTensor]:

        # if the tensor being added is also private
        if isinstance(other, NDimEntityPhiTensor):
            if self.entities != other.entities:
                return self.gamma + other.gamma

            return NDimEntityPhiTensor(
                child=self.child + other.child,
                min_vals=self.min_vals + other.min_vals,
                max_vals=self.max_vals + other.max_vals,
                entities=self.entities,
                # scalar_manager=self.scalar_manager,
            )

        # if the tensor being added is a public tensor / int / float / etc.
        elif is_acceptable_simple_type(other):
            return NDimEntityPhiTensor(
                child=self.child + other,
                min_vals=self.min_vals + other,
                max_vals=self.max_vals + other,
                entities=self.entities,
                # scalar_manager=self.scalar_manager,
            )

        elif isinstance(other, IntermediateGammaTensor):
            return self.gamma + other
        else:
            print("Type is unsupported:" + str(type(other)))
            raise NotImplementedError

    def sum(
        self, axis: Optional[Union[int, Tuple[int, ...]]] = None
    ) -> Union[NDimEntityPhiTensor, GammaTensor]:
        # TODO: Add support for axes arguments later
        if len(self.entities.one_hot_lookup) == 1:
            return NDimEntityPhiTensor(
                child=self.child.sum(),
                min_vals=self.min_vals.sum(axis=None),
                max_vals=self.max_vals.sum(axis=None),
                entities=DataSubjectList.from_objs(
                    self.entities.one_hot_lookup[0]
                ),  # Need to check this
            )

        return GammaTensor(
            value=np.array(self.child.sum()),
            data_subjects=self.entities.sum(),
            min_val=float(self.min_vals.sum(axis=None)),
            max_val=float(self.max_vals.sum(axis=None)),
            inputs=self.child,
        )

    def __ne__(  # type: ignore
        self, other: Any
    ) -> Union[NDimEntityPhiTensor, IntermediateGammaTensor, GammaTensor]:
        # TODO: what about entities and min / max values?
        if is_acceptable_simple_type(other) or len(self.child) == len(other.child):
            gamma_output = False
            if is_acceptable_simple_type(other):
                result = self.child != other
            else:
                # check entities match, if they dont gamma_output = True
                #
                result = self.child != other.child
                if isinstance(result, InitialGammaTensor):
                    gamma_output = True
            if not gamma_output:
                return self.copy_with(child=result)
            else:
                return self.copy_with(child=result).gamma
        else:
            raise Exception(
                "Tensor dims do not match for __eq__: "
                + f"{len(self.child)} != {len(other.child)}"
            )

    def __neg__(self) -> NDimEntityPhiTensor:

        return NDimEntityPhiTensor(
            child=self.child * -1,
            min_vals=self.max_vals * -1,
            max_vals=self.min_vals * -1,
            entities=self.entities,
        )

    def _object2bytes(self) -> bytes:
        schema = get_capnp_schema(schema_file="ndept.capnp")

        ndept_struct: CapnpModule = schema.NDEPT  # type: ignore
        ndept_msg = ndept_struct.new_message()
        # this is how we dispatch correct deserialization of bytes
        ndept_msg.magicHeader = serde_magic_header(type(self))

        ndept_msg.child = capnp_serialize(self.child)
        ndept_msg.minVals = capnp_serialize(self.min_vals.data)
        ndept_msg.maxVals = capnp_serialize(self.max_vals.data)
        ndept_msg.dataSubjectsIndexed = capnp_serialize(
            self.entities.data_subjects_indexed
        )

        ndept_msg.oneHotLookup = capnp_serialize(
            liststrtonumpyutf8(self.entities.one_hot_lookup)
        )

        # to pack or not to pack?
        # to_bytes = ndept_msg.to_bytes()

        return ndept_msg.to_bytes_packed()

    @staticmethod
    def _bytes2object(buf: bytes) -> NDimEntityPhiTensor:
        schema = get_capnp_schema(schema_file="ndept.capnp")
        ndept_struct: CapnpModule = schema.NDEPT  # type: ignore
        # https://stackoverflow.com/questions/48458839/capnproto-maximum-filesize
        MAX_TRAVERSAL_LIMIT = 2**64 - 1
        # to pack or not to pack?
        # ndept_msg = ndept_struct.from_bytes(buf, traversal_limit_in_words=2 ** 64 - 1)
        ndept_msg = ndept_struct.from_bytes_packed(
            buf, traversal_limit_in_words=MAX_TRAVERSAL_LIMIT
        )

        child = capnp_deserialize(ndept_msg.child)
        min_vals = capnp_deserialize(ndept_msg.minVals)
        max_vals = capnp_deserialize(ndept_msg.maxVals)
        data_subjects_indexed = capnp_deserialize(ndept_msg.dataSubjectsIndexed)
        one_hot_lookup = numpyutf8tolist(capnp_deserialize(ndept_msg.oneHotLookup))

        entity_list = DataSubjectList(one_hot_lookup, data_subjects_indexed)

        return NDimEntityPhiTensor(
            child=child, min_vals=min_vals, max_vals=max_vals, entities=entity_list
        )
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
#  IkaLog
#  ======
#  Copyright (C) 2015 Takeshi HASEGAWA
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
import numpy as np
import cv2
import time
import sys
from IkaUtils import *

## Tracker the control tower, (or rainmaker)
#
class IkaScene_TowerTracker:
	# 720p サイズでの値
	tower_width = 580
	tower_left = 1280 / 2 - tower_width / 2
	tower_top = 78
	tower_height = 88

	tower_line_top = 93
	tower_line_height = 5

	def reset(self, context):
		context['game']['tower'] = {
			'pos': 0,
			'max': 0,
			'min': 0,
		}

	def towerPos(self, context):
		img = context['engine']['frame'][self.tower_line_top:self.tower_line_top + self.tower_line_height, self.tower_left:self.tower_left + self.tower_width]
		img2 = cv2.resize(img, (self.tower_width, 100))
		img_hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
		for i in range(2):
			img2[20:40,:, i] = cv2.resize(img_hsv[:,:,0], (self.tower_width, 20))
			img2[40:60,:, i] = cv2.resize(img_hsv[:,:,1], (self.tower_width, 20))
			img2[60:80,:, i] = cv2.resize(img_hsv[:,:,2], (self.tower_width, 20))

		# ゲージのうち信頼できる部分だけでマスクする
		img3 = np.minimum(img, self.ui_tower_mask)

		# 白い部分にいまヤグラ/ホコがある
		img3_hsv = cv2.cvtColor(img3, cv2.COLOR_BGR2HSV)
		white_mask_s = cv2.inRange(img3_hsv[:, :, 1], 0, 8)
		white_mask_v = cv2.inRange(img3_hsv[:, :, 2], 248, 256)
		white_mask = np.minimum(white_mask_s, white_mask_v)
		x_list = np.arange(self.tower_width)
		tower_x = np.extract(white_mask[3, :] > 128, x_list)
		tower_xPos = np.average(tower_x)

		# FixMe: マスクした関係が位置がずれている可能性があるので、適宜補正すべき

		xPos_pct = (tower_xPos - self.tower_width / 2) / (self.tower_width * 0.86 / 2) * 100

		# あきらかにおかしい値が出たらとりあえず排除
		if xPos_pct < -120 or 120 < xPos_pct:
			xPos_pct = Nan

		return xPos_pct

	def match(self, context):
		if context['game']['rule'] is None:
			return None

		if not (context['game']['rule']['name'] != 'ガチヤグラ' and context['game']['rule']['name'] != 'ガチホコバトル'):
			return None

		xPos_pct = self.towerPos(context)

		if xPos_pct != xPos_pct:
			# 値がとれていない
			xPos_pct = context['game']['tower']['pos']

		xPos_pct = int(xPos_pct)

		context['game']['tower']['pos'] = xPos_pct
		context['game']['tower']['min'] = min(xPos_pct, context['game']['tower']['min'])
		context['game']['tower']['max'] = max(xPos_pct, context['game']['tower']['max'])
		return context['game']['tower']

	def __init__(self):
		self.ui_tower_mask = cv2.imread('masks/ui_tower.png')
		self.ui_tower_mask = self.ui_tower_mask[self.tower_line_top:self.tower_line_top + self.tower_line_height, self.tower_left:self.tower_left + self.tower_width]

if __name__ == "__main__":
	target = cv2.imread(sys.argv[1])
#	target = cv2.resize(target, (1280, 720))
	obj = IkaScene_RankedBattle()

	context = {
		'engine': { 'frame': target },
		'game': {},
	}

	r = obj.match(context)
	cv2.waitKey()
<commit_msg>IkaScene_TowerTracker: Check game mode in unicode.<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
#  IkaLog
#  ======
#  Copyright (C) 2015 Takeshi HASEGAWA
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
import numpy as np
import cv2
import time
import sys
from IkaUtils import *

## Tracker the control tower, (or rainmaker)
#
class IkaScene_TowerTracker:
	# 720p サイズでの値
	tower_width = 580
	tower_left = 1280 / 2 - tower_width / 2
	tower_top = 78
	tower_height = 88

	tower_line_top = 93
	tower_line_height = 5

	def reset(self, context):
		context['game']['tower'] = {
			'pos': 0,
			'max': 0,
			'min': 0,
		}

	def towerPos(self, context):
		img = context['engine']['frame'][self.tower_line_top:self.tower_line_top + self.tower_line_height, self.tower_left:self.tower_left + self.tower_width]
		img2 = cv2.resize(img, (self.tower_width, 100))
		img_hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
		for i in range(2):
			img2[20:40,:, i] = cv2.resize(img_hsv[:,:,0], (self.tower_width, 20))
			img2[40:60,:, i] = cv2.resize(img_hsv[:,:,1], (self.tower_width, 20))
			img2[60:80,:, i] = cv2.resize(img_hsv[:,:,2], (self.tower_width, 20))

		# ゲージのうち信頼できる部分だけでマスクする
		img3 = np.minimum(img, self.ui_tower_mask)

		# 白い部分にいまヤグラ/ホコがある
		img3_hsv = cv2.cvtColor(img3, cv2.COLOR_BGR2HSV)
		white_mask_s = cv2.inRange(img3_hsv[:, :, 1], 0, 8)
		white_mask_v = cv2.inRange(img3_hsv[:, :, 2], 248, 256)
		white_mask = np.minimum(white_mask_s, white_mask_v)
		x_list = np.arange(self.tower_width)
		tower_x = np.extract(white_mask[3, :] > 128, x_list)
		tower_xPos = np.average(tower_x)

		# FixMe: マスクした関係が位置がずれている可能性があるので、適宜補正すべき

		xPos_pct = (tower_xPos - self.tower_width / 2) / (self.tower_width * 0.86 / 2) * 100

		# あきらかにおかしい値が出たらとりあえず排除
		if xPos_pct < -120 or 120 < xPos_pct:
			xPos_pct = Nan

		return xPos_pct

	def match(self, context):
		if context['game']['rule'] is None:
			return None

		applicable_modes = [u'ガチヤグラ', u'ガチホコバトル', 'ガチヤグラ', 'ガチホコバトル']
		if not (context['game']['rule']['name'] in applicable_modes):
			return None

		xPos_pct = self.towerPos(context)

		if xPos_pct != xPos_pct:
			# 値がとれていない
			xPos_pct = context['game']['tower']['pos']

		xPos_pct = int(xPos_pct)

		context['game']['tower']['pos'] = xPos_pct
		context['game']['tower']['min'] = min(xPos_pct, context['game']['tower']['min'])
		context['game']['tower']['max'] = max(xPos_pct, context['game']['tower']['max'])
		return context['game']['tower']

	def __init__(self):
		self.ui_tower_mask = cv2.imread('masks/ui_tower.png')
		self.ui_tower_mask = self.ui_tower_mask[self.tower_line_top:self.tower_line_top + self.tower_line_height, self.tower_left:self.tower_left + self.tower_width]

if __name__ == "__main__":
	target = cv2.imread(sys.argv[1])
#	target = cv2.resize(target, (1280, 720))
	obj = IkaScene_RankedBattle()

	context = {
		'engine': { 'frame': target },
		'game': {},
	}

	r = obj.match(context)
	cv2.waitKey()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Forms for the Instanssi store.

import time
import random
import hashlib
import logging

from django import forms
from django.core.urlresolvers import reverse
from django.db import IntegrityError
from crispy_forms.helper import FormHelper
from crispy_forms.layout import \
    Submit, Layout, Fieldset, ButtonHolder, Hidden, Div, HTML
from Instanssi.store.models import StoreItem, TransactionItem, StoreTransaction

# Logger
logger = logging.getLogger(__name__)

# for creating ticket key hash
def gen_sha(text):
    h = hashlib.sha1()
    h.update(text)
    return h.hexdigest()

class StoreOrderForm(forms.ModelForm):
    """Displays an order form with items for a specific event listed."""

    email_confirm = forms.EmailField(
        label=u'Vahvista sähköposti', max_length=255
    )

    read_terms = forms.BooleanField(
        label=u'Hyväksyn toimitusehdot', required=True
    )

    def __init__(self, *args, **kwargs):
        self.event_id = kwargs.pop('event_id', None)
        super(StoreOrderForm, self).__init__(*args, **kwargs)

        self.helper = FormHelper()
        self.helper.form_class = 'store'

        self.fields['read_terms'].help_text = \
            u'Olen lukenut toimitusehdot ' \
            u'ja hyväksyn ne. (Luethan myös rekisteriselosteen)' % (reverse('store:terms'), reverse('store:privacy'))

        item_fields = Fieldset(u'', css_class='store-items')
        for item in StoreItem.items_for_event(self.event_id):
            name = 'item-%s' % item.id
            self.fields[name] = forms.IntegerField(
                initial=0, min_value=0, max_value=item.num_available(),
                label=u'%s' % (item.name),
                help_text=item.description,
                required=False
            )

            # Set data-maxvalue attr for field, just for javascript stuff
            self.fields[name].widget.attrs['data-maxvalue'] = item.num_available()

            # Container
            mdiv = Div()

            # Print message if item is sold out
            if item.num_in_store() <= 0:
                self.fields[name].widget.attrs['disabled'] = True
                self.fields[name].label += u' (Lopussa)'

            # Print img tag if item has image
            if item.imagefile_thumbnail:
                mdiv.fields.append(
                    HTML(''
                         % (item.imagefile_thumbnail.url, item.imagefile_original.url)),
                )

            mdiv.fields.append(
                HTML(u'%d €/kpl' % item.price)
            )
            mdiv.fields.append(name)
            item_fields.fields.append(mdiv)

        self.helper.layout = Layout(
            HTML(u'
Tuotteet
'),
            item_fields,
            HTML(u'
Maksajan tiedot
'),
            Fieldset(
                u'',
                'firstname',
                'lastname',
                'email',
                'email_confirm',
                'telephone',
                'mobile',
                'company',
                'street',
                'postalcode',
                'city',
                'country',
                'information',
                'read_terms',
                ButtonHolder(
                    Submit('Buy', u'Osta')
                ),
                css_class='store-details'
            )
        )

    def _dataitems(self):
        for key, value in self.data.iteritems():
            try:
                if key.startswith('item-') and int(value):
                    yield (key[5:], int(value))
            except:
                continue

    def clean(self):
        cleaned_data = super(StoreOrderForm, self).clean()
        total_items = 0

        fails = []

        # also check that the purchase amount for each field makes sense
        for (item_id, amount) in self._dataitems():
            store_item = StoreItem.objects.get(id=int(item_id))
            total_items += amount
            if store_item.num_available() < amount:
                fails.append(
                    u'Tuotetta "%s" ei ole saatavilla riittävästi!'
                    % store_item.name
                )

        if not total_items:
            fails.append(u'Tilauksessa on oltava ainakin yksi tuote!')

        if 'email' in self.data and 'email_confirm' in self.data:
            if self.data['email'] != self.data['email_confirm']:
                if not 'email_confirm' in self._errors:
                    self._errors['email_confirm'] = self.error_class()
                self._errors['email_confirm'].append(
                    u'Osoitteet eivät täsmää!'
                )
                fails.append(
                    u'Vahvista sähköpostiosoitteesi kirjoittamalla sama '
                    u'osoite molempiin kenttiin!'
                )

        if fails:
            raise forms.ValidationError(fails)

        return cleaned_data

    def save(self, commit=True):
        """Saves a store transaction form, also generating TransactionItems
        for each item in the post data."""

        new_transaction = super(StoreOrderForm, self).save(commit=False)

        for i in range(10):
            try:
                new_transaction.key = gen_sha('%s|%s|%s|%s|%s|%s' % (i, new_transaction.firstname, new_transaction.lastname, time.time(), random.random(), random.random()))
                new_transaction.save()
                break
            except IntegrityError as ex:
                logger.warning("SHA-1 Collision in transaction (WTF!) Key: %s, exception: %s." % (ta.key, ex))

        transaction_items = []
        for (item_id, amount) in self._dataitems():
            if amount > 0:
                store_item = StoreItem.objects.get(id=int(item_id))
                new_item = TransactionItem(
                    item=store_item,
                    transaction=new_transaction,
                    amount=amount
                )
                new_item.save()
                transaction_items.append(new_item)

        return new_transaction

    class Meta:
        model = StoreTransaction
        exclude = ('time_created', 'time_paid', 'token', 'paid', 'key', 'status')
<commit_msg>store: Fix unicode errors<commit_after># -*- coding: utf-8 -*-
# Forms for the Instanssi store.

import time
import random
import hashlib
import logging

from django import forms
from django.core.urlresolvers import reverse
from django.db import IntegrityError
from crispy_forms.helper import FormHelper
from crispy_forms.layout import \
    Submit, Layout, Fieldset, ButtonHolder, Hidden, Div, HTML
from Instanssi.store.models import StoreItem, TransactionItem, StoreTransaction

# Logger
logger = logging.getLogger(__name__)

# for creating ticket key hash
def gen_sha(text):
    h = hashlib.sha1()
    h.update(text)
    return h.hexdigest()

class StoreOrderForm(forms.ModelForm):
    """Displays an order form with items for a specific event listed."""

    email_confirm = forms.EmailField(
        label=u'Vahvista sähköposti', max_length=255
    )

    read_terms = forms.BooleanField(
        label=u'Hyväksyn toimitusehdot', required=True
    )

    def __init__(self, *args, **kwargs):
        self.event_id = kwargs.pop('event_id', None)
        super(StoreOrderForm, self).__init__(*args, **kwargs)

        self.helper = FormHelper()
        self.helper.form_class = 'store'

        self.fields['read_terms'].help_text = \
            u'Olen lukenut toimitusehdot ' \
            u'ja hyväksyn ne. (Luethan myös rekisteriselosteen)' % (reverse('store:terms'), reverse('store:privacy'))

        item_fields = Fieldset(u'', css_class='store-items')
        for item in StoreItem.items_for_event(self.event_id):
            name = 'item-%s' % item.id
            self.fields[name] = forms.IntegerField(
                initial=0, min_value=0, max_value=item.num_available(),
                label=u'%s' % (item.name),
                help_text=item.description,
                required=False
            )

            # Set data-maxvalue attr for field, just for javascript stuff
            self.fields[name].widget.attrs['data-maxvalue'] = item.num_available()

            # Container
            mdiv = Div()

            # Print message if item is sold out
            if item.num_in_store() <= 0:
                self.fields[name].widget.attrs['disabled'] = True
                self.fields[name].label += u' (Lopussa)'

            # Print img tag if item has image
            if item.imagefile_thumbnail:
                mdiv.fields.append(
                    HTML(''
                         % (item.imagefile_thumbnail.url, item.imagefile_original.url)),
                )

            mdiv.fields.append(
                HTML(u'%d €/kpl' % item.price)
            )
            mdiv.fields.append(name)
            item_fields.fields.append(mdiv)

        self.helper.layout = Layout(
            HTML(u'
Tuotteet
'),
            item_fields,
            HTML(u'
Maksajan tiedot
'),
            Fieldset(
                u'',
                'firstname',
                'lastname',
                'email',
                'email_confirm',
                'telephone',
                'mobile',
                'company',
                'street',
                'postalcode',
                'city',
                'country',
                'information',
                'read_terms',
                ButtonHolder(
                    Submit('Buy', u'Osta')
                ),
                css_class='store-details'
            )
        )

    def _dataitems(self):
        for key, value in self.data.iteritems():
            try:
                if key.startswith('item-') and int(value):
                    yield (key[5:], int(value))
            except:
                continue

    def clean(self):
        cleaned_data = super(StoreOrderForm, self).clean()
        total_items = 0

        fails = []

        # also check that the purchase amount for each field makes sense
        for (item_id, amount) in self._dataitems():
            store_item = StoreItem.objects.get(id=int(item_id))
            total_items += amount
            if store_item.num_available() < amount:
                fails.append(
                    u'Tuotetta "%s" ei ole saatavilla riittävästi!'
                    % store_item.name
                )

        if not total_items:
            fails.append(u'Tilauksessa on oltava ainakin yksi tuote!')

        if 'email' in self.data and 'email_confirm' in self.data:
            if self.data['email'] != self.data['email_confirm']:
                if not 'email_confirm' in self._errors:
                    self._errors['email_confirm'] = self.error_class()
                self._errors['email_confirm'].append(
                    u'Osoitteet eivät täsmää!'
                )
                fails.append(
                    u'Vahvista sähköpostiosoitteesi kirjoittamalla sama '
                    u'osoite molempiin kenttiin!'
                )

        if fails:
            raise forms.ValidationError(fails)

        return cleaned_data

    def save(self, commit=True):
        """Saves a store transaction form, also generating TransactionItems
        for each item in the post data."""

        new_transaction = super(StoreOrderForm, self).save(commit=False)

        for i in range(10):
            try:
                new_transaction.key = gen_sha(u'%s|%s|%s|%s|%s|%s' % (i, new_transaction.firstname, new_transaction.lastname, time.time(), random.random(), random.random()))
                new_transaction.save()
                break
            except IntegrityError as ex:
                logger.warning("SHA-1 Collision in transaction (WTF!) Key: %s, exception: %s." % (ta.key, ex))

        transaction_items = []
        for (item_id, amount) in self._dataitems():
            if amount > 0:
                store_item = StoreItem.objects.get(id=int(item_id))
                new_item = TransactionItem(
                    item=store_item,
                    transaction=new_transaction,
                    amount=amount
                )
                new_item.save()
                transaction_items.append(new_item)

        return new_transaction

    class Meta:
        model = StoreTransaction
        exclude = ('time_created', 'time_paid', 'token', 'paid', 'key', 'status')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from common.auth import user_access_required
from django.shortcuts import render_to_response
from django.http import HttpResponseRedirect
from django.template import RequestContext
from django.contrib import auth
from django.core.urlresolvers import reverse
from Instanssi.users.forms import OpenIDLoginForm, DjangoLoginForm, ProfileForm
from common.misc import get_url_local_path

AUTH_METHODS = [
    # Short name, social-auth, friendly name
    ('facebook', 'facebook', 'Facebook'),
    ('google', 'google-oauth2', 'Google'),
    ('twitter', 'twitter', 'Twitter'),
    ('github', 'github', 'Github'),
    ('battlenet', 'battlenet-oauth2', 'Battle.net'),
    ('steam', 'steam', 'Steam'),
]

def login(request):
    if request.user.is_authenticated():
        return HttpResponseRedirect(reverse('users:profile'))

    # Get referer for redirect
    # Make sure that the referrer is a local path.
    if 'next' in request.GET:
        next_page = get_url_local_path(request.GET['next'])
    else:
        next_page = get_url_local_path(request.META.get('HTTP_REFERER', reverse('users:profile')))

    # Test django login form
    if request.method == "POST":
        djangoform = DjangoLoginForm(request.POST)
        if djangoform.is_valid():
            djangoform.login(request)
            return HttpResponseRedirect(djangoform.cleaned_data['next'])
    else:
        djangoform = DjangoLoginForm(next=next_page)

    # Openid login form
    # The form will be handled elsewhere; this is only for rendering the form.
    openidform = OpenIDLoginForm(next=next_page)

    # Render response
    return render_to_response("users/login.html", {
        'djangoform': djangoform,
        'openidform': openidform,
        'next': next_page,
        'AUTH_METHODS': AUTH_METHODS
    }, context_instance=RequestContext(request))

def loggedout(request):
    return render_to_response("users/loggedout.html")

@user_access_required
def profile(request):
    from social.apps.django_app.default.models import DjangoStorage

    if request.method == "POST":
        profileform = ProfileForm(request.POST, instance=request.user, user=request.user)
        if profileform.is_valid():
            profileform.save()
            return HttpResponseRedirect(reverse('users:profile'))
    else:
        profileform = ProfileForm(instance=request.user, user=request.user)

    # Get all active providers for this user
    active_providers = []
    for social_auth in DjangoStorage.user.get_social_auth_for_user(request.user):
        active_providers.append(social_auth.provider)

    # Providers list
    methods = []
    for method in AUTH_METHODS:
        methods.append(method + (method[1] in active_providers, ))

    return render_to_response("users/profile.html", {
        'profileform': profileform,
        'active_providers': active_providers,
        'AUTH_METHODS': methods
    }, context_instance=RequestContext(request))

def logout(request):
    auth.logout(request)
    return HttpResponseRedirect(reverse('users:loggedout'))
<commit_msg>users: Fix for django 1.10<commit_after># -*- coding: utf-8 -*-

from common.auth import user_access_required
from django.shortcuts import render
from django.http import HttpResponseRedirect
from django.contrib import auth
from django.core.urlresolvers import reverse
from Instanssi.users.forms import OpenIDLoginForm, DjangoLoginForm, ProfileForm
from common.misc import get_url_local_path

AUTH_METHODS = [
    # Short name, social-auth, friendly name
    ('facebook', 'facebook', 'Facebook'),
    ('google', 'google-oauth2', 'Google'),
    ('twitter', 'twitter', 'Twitter'),
    ('github', 'github', 'Github'),
    ('battlenet', 'battlenet-oauth2', 'Battle.net'),
    ('steam', 'steam', 'Steam'),
]

def login(request):
    if request.user.is_authenticated():
        return HttpResponseRedirect(reverse('users:profile'))

    # Get referer for redirect
    # Make sure that the referrer is a local path.
    if 'next' in request.GET:
        next_page = get_url_local_path(request.GET['next'])
    else:
        next_page = get_url_local_path(request.META.get('HTTP_REFERER', reverse('users:profile')))

    # Test django login form
    if request.method == "POST":
        djangoform = DjangoLoginForm(request.POST)
        if djangoform.is_valid():
            djangoform.login(request)
            return HttpResponseRedirect(djangoform.cleaned_data['next'])
    else:
        djangoform = DjangoLoginForm(next=next_page)

    # Openid login form
    # The form will be handled elsewhere; this is only for rendering the form.
    openidform = OpenIDLoginForm(next=next_page)

    # Render response
    return render(request, "users/login.html", {
        'djangoform': djangoform,
        'openidform': openidform,
        'next': next_page,
        'AUTH_METHODS': AUTH_METHODS
    })

def loggedout(request):
    return render(request, "users/loggedout.html")

@user_access_required
def profile(request):
    from social.apps.django_app.default.models import DjangoStorage

    if request.method == "POST":
        profileform = ProfileForm(request.POST, instance=request.user, user=request.user)
        if profileform.is_valid():
            profileform.save()
            return HttpResponseRedirect(reverse('users:profile'))
    else:
        profileform = ProfileForm(instance=request.user, user=request.user)

    # Get all active providers for this user
    active_providers = []
    for social_auth in DjangoStorage.user.get_social_auth_for_user(request.user):
        active_providers.append(social_auth.provider)

    # Providers list
    methods = []
    for method in AUTH_METHODS:
        methods.append(method + (method[1] in active_providers, ))

    return render(request, "users/profile.html", {
        'profileform': profileform,
        'active_providers': active_providers,
        'AUTH_METHODS': methods
    })

def logout(request):
    auth.logout(request)
    return HttpResponseRedirect(reverse('users:loggedout'))
<|endoftext|>