<commit_before># Copyright 2016 The Johns Hopkins University Applied Physics Laboratory
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from six.moves import input
import logging
import datetime
import json
import time
from ..utils.log import always_log_info
import os
from math import floor
import random
from .config import Configuration, ConfigFileError
from collections import deque

class Engine(object):
    def __init__(self, config_file=None, backend_api_token=None, ingest_job_id=None, configuration=None):
        """
        A class to implement the core upload client workflow engine

        Args:
            config_file (str): Absolute path to a config file
            ingest_job_id (int): ID of the ingest job you want to work on
            backend_api_token (str): The authorization token for the Backend if used
            configuration(ingestclient.core.config.Configuration): A pre-loaded configuration instance
        """
        self.config = None
        self.msg_wait_iterations = 20  # Each iteration waits for 10 seconds for incoming messages
        self.backend = None
        self.validator = None
        self.tile_processor = None
        self.path_processor = None
        self.backend_api_token = backend_api_token
        self.credential_create_time = None

        # Properties of ingest after creation
        self.credentials = None
        self.ingest_job_id = ingest_job_id
        self.upload_job_queue = None
        self.job_status = 0
        self.tile_bucket = None
        self.job_params = None
        self.tile_count = 0
        self.access_denied = False
        self.access_denied_count = 0
        self.invalid_access_key = False
        self.invalid_access_key_count = 0

        if configuration:
            self.configure(configuration)
        elif config_file:
            self.configure_from_file(config_file)

    def configure_from_file(self, config_file):
        """
        Method to load a configuration file and setup the workflow engine
        Args:
            config_file (str): Absolute path to a config file

        Returns:
            None
        """
        try:
            with open(config_file, 'r') as file_handle:
                config_data = json.load(file_handle)
        except ValueError as _:
            # Bad json file
            raise ConfigFileError(
                "Malformed JSON in Ingest Configuration File.  Please double check contents and try again")
        except IOError as _:
            # File not found - python2/3 are different for missing files so us OSError
            raise ConfigFileError(
                "Ingest Configuration File not found.  Double check the provided path: {}".format(config_file))
        except OSError as _:
            # File not found - python2/3 are different for missing files so us OSError
            raise ConfigFileError(
                "Ingest Configuration File not found.  Double check the provided path: {}".format(config_file))

        self.configure(Configuration(config_data))

    def configure(self, configuration):
        """
        Method to apply a configuration and setup the workflow engine
        Args:
            configuration (Configuration)

        Returns:
            None
        """

        # Load Config file and validate
        self.config = configuration
        self.config.load_plugins()

        # Get backend
        self.backend = self.config.get_backend(self.backend_api_token)

        # Get validator and set config
        self.validator = self.config.get_validator()
        self.validator.schema = self.config.schema

        # Setup tile processor
        self.tile_processor = self.config.tile_processor_class
        self.tile_processor.setup(self.config.get_tile_processor_params())

        # Setup path processor
        self.path_processor = self.config.path_processor_class
        self.path_processor.setup(self.config.get_path_processor_params())

    def setup(self):
        """Method to setup the Engine by finishing configuring subclasses and validating the schema"""
        logger = logging.getLogger('ingest-client')

        msgs = self.validator.validate()

        for msg in msgs["info"]:
            logger.info(msg)

        if msgs["error"]:
            for msg in msgs["error"]:
                logger.info(msg)
            raise Exception("Validation Failed: {}".format(" - ".join(msgs["error"])))

        return msgs["question"]

    def create_job(self):
        """
        Method to create an ingest job

        Args:

        Returns:
            None

        """
        self.ingest_job_id = self.backend.create(self.config.config_data)

        always_log_info("CREATED INGEST JOB: {}".format(self.ingest_job_id))

    def join(self):
        """
        Method to join an ingest job upload

        Job Status: {0: Preparing, 1: Uploading, 2: Complete}

        Args:

        Returns:
            None

        """
        self.job_status, self.credentials, self.upload_job_queue, self.tile_bucket, self.job_params, self.tile_count = self.backend.join(self.ingest_job_id)

        # Set cred time
        self.credential_create_time = datetime.datetime.now()
        always_log_info("(pid={}) JOINED INGEST JOB: {}".format(os.getpid(), self.ingest_job_id))

    def cancel(self):
        """
        Method to cancel an ingest job

        Args:

        Returns:
            None

        """
        self.backend.cancel(self.ingest_job_id)

    def complete(self):
        """
        Method to complete an ingest job

        Args:

        Returns:
            None

        """
        self.backend.complete(self.ingest_job_id)

    def monitor(self, workers):
        """Method to monitor the progress of the ingest job

        Returns:
            None
        """
        logger = logging.getLogger('ingest-client')
        tile_rate_samples = deque(maxlen=6)
        last_task_count = None
        start_time = time.time()
        print_time = time.time()
        avg_tile_rate = 0
        while True:
            total_seconds = (datetime.datetime.now() - self.credential_create_time).total_seconds()
            if total_seconds > self.backend.credential_timeout:
                logger.warning("(pid={}) Credentials are expiring soon, attempting to renew credentials".format(
                    os.getpid()))
                self.join()
                always_log_info("(pid={}) Credentials refreshed successfully".format(os.getpid()))

            status = self.backend.get_job_status(self.ingest_job_id)
            if status:
                if last_task_count is None:
                    last_task_count = status["current_message_count"]
                    continue

                tile_rate_samples.append(last_task_count - status["current_message_count"])
                last_task_count = status["current_message_count"]

                avg_tile_rate = sum(tile_rate_samples) / float(len(tile_rate_samples))

            if (time.time() - print_time) > 30:
                print_time = time.time()
                # Print an update every 30 seconds
                if status:
                    if status["current_message_count"] != 0:
                        log_str = "Uploading ~{:.2f} tiles/min".format(avg_tile_rate * 6)
                        log_str += " - Approx {:d} of {:d} tiles remaining".format(status["current_message_count"],
                                                                                status["total_message_count"])
                        log_str += " - Elapsed time {:.2f} minutes".format((time.time() - start_time) / 60)
                        always_log_info(log_str)
                    else:
                        always_log_info(
                            "Waiting to ensure all upload tasks have been processed. Just a few minutes longer...")

                else:
                    always_log_info("Uploading in progress: Elapsed time {:.2f} minutes".format(
                        (time.time() - start_time) / 60))

            # Wait to loop
            time.sleep(10)

            # Check to see if worker processes have all ended
            alive_cnt = 0
            for worker in workers:
                if worker[0].is_alive():
                    alive_cnt += 1

            if alive_cnt == 0:
                # if no processes are alive you are done (or something broke)! Bail.
                break

    def run(self):
        """Method to run the upload loop

        Returns:

        """
        # Set up logger
        logger = logging.getLogger('ingest-client')

        # Make sure you are joined
        if not self.credentials:
            msg = "(pid={}) Cannot start ingest engine.  Credentials not successfully received from the ingest service.".format(os.getpid())
            logger.error(msg)
            raise Exception(msg)

        if self.job_status == 0:
            msg = "(pid={}) Cannot start ingest engine.  Ingest job is not ready yet.".format(os.getpid())
            logger.error(msg)
            raise Exception(msg)

        if self.job_status == 2:
            msg = "(pid={}) Ingest job already completed. Skipping ingest engine start.".format(os.getpid())
            logger.warning(msg)
            raise Exception(msg)

        # Do some work
        self.access_denied = False
        self.access_denied_count = 0
        self.invalid_access_key = False
        self.invalid_access_key_count = 0

        wait_cnt = 0
        while True:
            if self.access_denied:
                self.access_denied = False
                self.credential_create_time = datetime.datetime.min
            if self.invalid_access_key:
                self.invalid_access_key = False
                if self.invalid_access_key_count % 5 == 4:
                    self.credential_create_time = datetime.datetime.min
            # Check if you need to renew credentials
            total_seconds = (datetime.datetime.now() - self.credential_create_time).total_seconds()
            if total_seconds > self.backend.credential_timeout:
                logger.warning("(pid={}) Credentials are expiring soon, attempting to renew credentials".format(
                    os.getpid()))
                self.join()
                always_log_info("(pid={}) Credentials refreshed successfully".format(os.getpid()))

            # Get a task
            message_id, receipt_handle, msg = self.backend.get_task()

            if not msg:
                time.sleep(10)
                wait_cnt += 1
                if wait_cnt < self.msg_wait_iterations:
                    continue
                else:
                    break

            wait_cnt = 0
            key_parts = self.backend.decode_tile_key(msg['tile_key'])
            logger.info("(pid={}) Processing Task -  X:{} Y:{} Z:{} T:{}".format(os.getpid(),
                                                                                key_parts["x_index"],
                                                                                key_parts["y_index"],
                                                                                key_parts["z_index"],
                                                                                key_parts["t_index"]))

            # Call path processor
            filename = self.path_processor.process(key_parts["x_index"],
                                                   key_parts["y_index"],
                                                   key_parts["z_index"],
                                                   key_parts["t_index"])

            # Call tile processor
            handle = self.tile_processor.process(filename,
                                                 key_parts["x_index"],
                                                 key_parts["y_index"],
                                                 key_parts["z_index"],
                                                 key_parts["t_index"])

            try:
                metadata = {'chunk_key': msg['chunk_key'],
                            'ingest_job': self.ingest_job_id,
                            'parameters': self.job_params,
                            }
                handle.seek(0)
                response = self.backend.bucket.put_object(ACL='private',
                                                          Body=handle,
                                                          Key=msg['tile_key'],
                                                          Metadata={
                                                              'message_id': message_id,
                                                              'receipt_handle': receipt_handle,
                                                              'metadata': json.dumps(metadata, separators=(',', ':'))
                                                          },
                                                          StorageClass='STANDARD')
                logger.info("(pid={}) Successfully wrote file: {}".format(os.getpid(), response.key))

            except Exception as e:
                logger.error("(pid={}) Upload Failed -  X:{} Y:{} Z:{} T:{} - {}".format(os.getpid(),
                                                                                key_parts["x_index"],
                                                                                key_parts["y_index"],
                                                                                key_parts["z_index"],
                                                                                key_parts["t_index"],
                                                                                e))
                if str(e).startswith("An error occurred (AccessDenied) when calling the PutObject operation"):
                    self.access_denied = True
                    self.access_denied_count += 1
                    if self.access_denied_count >= 20:
                        logger.error("(pid={}) failed 20 times with same error, breaking out of loop: {} ".format(
                            os.getpid(), e))
                        break
                elif str(e).startswith("An error occurred (InvalidAccessKeyId) when calling the PutObject operation"):
                    time.sleep(5)
                    self.invalid_access_key = True
                    self.invalid_access_key_count += 1
                    if self.invalid_access_key_count >= 20:
                        logger.error("(pid={}) failed 20 times with same error, breaking out of loop: {} ".format(
                            os.getpid(), e))
                        break
<commit_msg>updated engine.py based off of comments for stress test bugfix PR#7<commit_after># Copyright 2016 The Johns Hopkins University Applied Physics Laboratory
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from six.moves import input
import logging
import datetime
import json
import time
from ..utils.log import always_log_info
import os
from math import floor
import random
from .config import Configuration, ConfigFileError
from collections import deque

class Engine(object):
    def __init__(self, config_file=None, backend_api_token=None, ingest_job_id=None, configuration=None):
        """
        A class to implement the core upload client workflow engine

        Args:
            config_file (str): Absolute path to a config file
            ingest_job_id (int): ID of the ingest job you want to work on
            backend_api_token (str): The authorization token for the Backend if used
            configuration(ingestclient.core.config.Configuration): A pre-loaded configuration instance
        """
        self.config = None
        self.msg_wait_iterations = 20  # Each iteration waits for 10 seconds for incoming messages
        self.backend = None
        self.validator = None
        self.tile_processor = None
        self.path_processor = None
        self.backend_api_token = backend_api_token
        self.credential_create_time = None

        # Properties of ingest after creation
        self.credentials = None
        self.ingest_job_id = ingest_job_id
        self.upload_job_queue = None
        self.job_status = 0
        self.tile_bucket = None
        self.job_params = None
        self.tile_count = 0
        self.access_denied = False
        self.access_denied_count = 0
        self.invalid_access_key = False
        self.invalid_access_key_count = 0

        if configuration:
            self.configure(configuration)
        elif config_file:
            self.configure_from_file(config_file)

    def configure_from_file(self, config_file):
        """
        Method to load a configuration file and setup the workflow engine
        Args:
            config_file (str): Absolute path to a config file

        Returns:
            None
        """
        try:
            with open(config_file, 'r') as file_handle:
                config_data = json.load(file_handle)
        except ValueError as _:
            # Bad json file
            raise ConfigFileError(
                "Malformed JSON in Ingest Configuration File.  Please double check contents and try again")
        except IOError as _:
            # File not found - python2/3 are different for missing files so us OSError
            raise ConfigFileError(
                "Ingest Configuration File not found.  Double check the provided path: {}".format(config_file))
        except OSError as _:
            # File not found - python2/3 are different for missing files so us OSError
            raise ConfigFileError(
                "Ingest Configuration File not found.  Double check the provided path: {}".format(config_file))

        self.configure(Configuration(config_data))

    def configure(self, configuration):
        """
        Method to apply a configuration and setup the workflow engine
        Args:
            configuration (Configuration)

        Returns:
            None
        """

        # Load Config file and validate
        self.config = configuration
        self.config.load_plugins()

        # Get backend
        self.backend = self.config.get_backend(self.backend_api_token)

        # Get validator and set config
        self.validator = self.config.get_validator()
        self.validator.schema = self.config.schema

        # Setup tile processor
        self.tile_processor = self.config.tile_processor_class
        self.tile_processor.setup(self.config.get_tile_processor_params())

        # Setup path processor
        self.path_processor = self.config.path_processor_class
        self.path_processor.setup(self.config.get_path_processor_params())

    def setup(self):
        """Method to setup the Engine by finishing configuring subclasses and validating the schema"""
        logger = logging.getLogger('ingest-client')

        msgs = self.validator.validate()

        for msg in msgs["info"]:
            logger.info(msg)

        if msgs["error"]:
            for msg in msgs["error"]:
                logger.info(msg)
            raise Exception("Validation Failed: {}".format(" - ".join(msgs["error"])))

        return msgs["question"]

    def create_job(self):
        """
        Method to create an ingest job

        Args:

        Returns:
            None

        """
        self.ingest_job_id = self.backend.create(self.config.config_data)

        always_log_info("CREATED INGEST JOB: {}".format(self.ingest_job_id))

    def join(self):
        """
        Method to join an ingest job upload

        Job Status: {0: Preparing, 1: Uploading, 2: Complete}

        Args:

        Returns:
            None

        """
        self.job_status, self.credentials, self.upload_job_queue, self.tile_bucket, self.job_params, self.tile_count = self.backend.join(self.ingest_job_id)

        # Set cred time
        self.credential_create_time = datetime.datetime.now()
        always_log_info("(pid={}) JOINED INGEST JOB: {}".format(os.getpid(), self.ingest_job_id))

    def cancel(self):
        """
        Method to cancel an ingest job

        Args:

        Returns:
            None

        """
        self.backend.cancel(self.ingest_job_id)

    def complete(self):
        """
        Method to complete an ingest job

        Args:

        Returns:
            None

        """
        self.backend.complete(self.ingest_job_id)

    def monitor(self, workers):
        """Method to monitor the progress of the ingest job

        Returns:
            None
        """
        logger = logging.getLogger('ingest-client')
        tile_rate_samples = deque(maxlen=6)
        last_task_count = None
        start_time = time.time()
        print_time = time.time()
        avg_tile_rate = 0
        while True:
            total_seconds = (datetime.datetime.now() - self.credential_create_time).total_seconds()
            if total_seconds > self.backend.credential_timeout:
                logger.warning("(pid={}) Credentials are expiring soon, attempting to renew credentials".format(
                    os.getpid()))
                self.join()
                always_log_info("(pid={}) Credentials refreshed successfully".format(os.getpid()))

            status = self.backend.get_job_status(self.ingest_job_id)
            if status:
                if last_task_count is None:
                    last_task_count = status["current_message_count"]
                    continue

                tile_rate_samples.append(last_task_count - status["current_message_count"])
                last_task_count = status["current_message_count"]

                avg_tile_rate = sum(tile_rate_samples) / float(len(tile_rate_samples))

            if (time.time() - print_time) > 30:
                print_time = time.time()
                # Print an update every 30 seconds
                if status:
                    if status["current_message_count"] != 0:
                        log_str = "Uploading ~{:.2f} tiles/min".format(avg_tile_rate * 6)
                        log_str += " - Approx {:d} of {:d} tiles remaining".format(status["current_message_count"],
                                                                                status["total_message_count"])
                        log_str += " - Elapsed time {:.2f} minutes".format((time.time() - start_time) / 60)
                        always_log_info(log_str)
                    else:
                        log_str = "Waiting to ensure all upload tasks have been processed. Just a few minutes longer..."
                        always_log_info(log_str)

                else:
                    log_str = "Uploading in progress: Elapsed time {:.2f} minutes"
                    log_str = log_str.format((time.time() - start_time) / 60)
                    always_log_info()

            # Wait to loop
            time.sleep(10)

            # Check to see if worker processes have all ended
            alive_cnt = 0
            for worker in workers:
                if worker[0].is_alive():
                    alive_cnt += 1

            if alive_cnt == 0:
                # if no processes are alive you are done (or something broke)! Bail.
                break

    def run(self):
        """Method to run the upload loop

        Returns:

        """
        # Set up logger
        logger = logging.getLogger('ingest-client')

        # Make sure you are joined
        if not self.credentials:
            msg = "(pid={}) Cannot start ingest engine.  Credentials not successfully received from the ingest service.".format(os.getpid())
            logger.error(msg)
            raise Exception(msg)

        if self.job_status == 0:
            msg = "(pid={}) Cannot start ingest engine.  Ingest job is not ready yet.".format(os.getpid())
            logger.error(msg)
            raise Exception(msg)

        if self.job_status == 2:
            msg = "(pid={}) Ingest job already completed. Skipping ingest engine start.".format(os.getpid())
            logger.warning(msg)
            raise Exception(msg)

        # Do some work
        self.access_denied = False
        self.access_denied_count = 0
        self.invalid_access_key = False
        self.invalid_access_key_count = 0

        wait_cnt = 0
        while True:
            if self.access_denied:
                self.access_denied = False
                self.credential_create_time = datetime.datetime.min
            if self.invalid_access_key:
                self.invalid_access_key = False
                if self.invalid_access_key_count % 5 == 4:
                    self.credential_create_time = datetime.datetime.min
            # Check if you need to renew credentials
            total_seconds = (datetime.datetime.now() - self.credential_create_time).total_seconds()
            if total_seconds > self.backend.credential_timeout:
                logger.warning("(pid={}) Credentials are expiring soon, attempting to renew credentials".format(
                    os.getpid()))
                self.join()
                always_log_info("(pid={}) Credentials refreshed successfully".format(os.getpid()))

            # Get a task
            message_id, receipt_handle, msg = self.backend.get_task()

            if not msg:
                time.sleep(10)
                wait_cnt += 1
                if wait_cnt < self.msg_wait_iterations:
                    continue
                else:
                    break

            wait_cnt = 0
            key_parts = self.backend.decode_tile_key(msg['tile_key'])
            logger.info("(pid={}) Processing Task -  X:{} Y:{} Z:{} T:{}".format(os.getpid(),
                                                                                key_parts["x_index"],
                                                                                key_parts["y_index"],
                                                                                key_parts["z_index"],
                                                                                key_parts["t_index"]))

            # Call path processor
            filename = self.path_processor.process(key_parts["x_index"],
                                                   key_parts["y_index"],
                                                   key_parts["z_index"],
                                                   key_parts["t_index"])

            # Call tile processor
            handle = self.tile_processor.process(filename,
                                                 key_parts["x_index"],
                                                 key_parts["y_index"],
                                                 key_parts["z_index"],
                                                 key_parts["t_index"])

            try:
                metadata = {'chunk_key': msg['chunk_key'],
                            'ingest_job': self.ingest_job_id,
                            'parameters': self.job_params,
                            }
                handle.seek(0)
                response = self.backend.bucket.put_object(ACL='private',
                                                          Body=handle,
                                                          Key=msg['tile_key'],
                                                          Metadata={
                                                              'message_id': message_id,
                                                              'receipt_handle': receipt_handle,
                                                              'metadata': json.dumps(metadata, separators=(',', ':'))
                                                          },
                                                          StorageClass='STANDARD')
                logger.info("(pid={}) Successfully wrote file: {}".format(os.getpid(), response.key))

            except Exception as e:
                logger.error("(pid={}) Upload Failed -  X:{} Y:{} Z:{} T:{} - {}".format(os.getpid(),
                                                                                key_parts["x_index"],
                                                                                key_parts["y_index"],
                                                                                key_parts["z_index"],
                                                                                key_parts["t_index"],
                                                                                e))
                if str(e).startswith("An error occurred (AccessDenied) when calling the PutObject operation"):
                    self.access_denied = True
                    self.access_denied_count += 1
                    if self.access_denied_count >= 20:
                        logger.error("(pid={}) failed 20 times with same error, breaking out of loop: {} ".format(
                            os.getpid(), e))
                        break
                elif str(e).startswith("An error occurred (InvalidAccessKeyId) when calling the PutObject operation"):
                    time.sleep(5)
                    self.invalid_access_key = True
                    self.invalid_access_key_count += 1
                    if self.invalid_access_key_count >= 20:
                        logger.error("(pid={}) failed 20 times with same error, breaking out of loop: {} ".format(
                            os.getpid(), e))
                        break
<|endoftext|>


<commit_before># coding: utf8

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import os.path
from decisionlib import DecisionTask

def main():
    decision = DecisionTask(
        project_name="Servo",  # Used in task names
        route_prefix="project.servo.servo",
        worker_type="servo-docker-worker",
    )

    # FIXME: remove this before merging in servo/servo
    os.environ["GIT_URL"] = "https://github.com/SimonSapin/servo"
    os.environ["GIT_REF"] = "refs/heads/taskcluster-experiments-20180920"
    os.environ["GIT_SHA"] = "a6dbfdd29f9b3f0ce0c13adc79fad99538a9a44b"
    decision.docker_image_cache_expiry = "1 week"
    decision.route_prefix = "project.servo.servo-taskcluster-experiments"
    # ~

    # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/caches
    cache_scopes = [
        "docker-worker:cache:cargo-*",
    ]
    build_caches = {
        "cargo-registry-cache": "/root/.cargo/registry",
        "cargo-git-cache": "/root/.cargo/git",
        "cargo-rustup": "/root/.rustup",
        "cargo-sccache": "/root/.cache/sccache",
    }
    build_artifacts_expiry = "1 week"
    build_env = {
        "RUST_BACKTRACE": "1",
        "RUSTFLAGS": "-Dwarnings",
        "CARGO_INCREMENTAL": "0",
        "SCCACHE_IDLE_TIMEOUT": "1200",
        "CCACHE": "sccache",
        "RUSTC_WRAPPER": "sccache",
    }
    build_kwargs = {
        "max_run_time_minutes": 60,
        "dockerfile": "build-x86_64-linux.dockerfile",
        "env": build_env,
        "scopes": cache_scopes,
        "cache": build_caches,
    }

    decision.create_task(
        task_name="Linux x86_64: tidy + dev build + unit tests",
        script="""
            ./mach test-tidy --no-progress --all
            ./mach build --dev
            ./mach test-unit
            ./mach package --dev
            ./mach test-tidy --no-progress --self-test
            python ./etc/memory_reports_over_time.py --test
            ./etc/ci/lockfile_changed.sh
            ./etc/ci/check_no_panic.sh
        """,
        **build_kwargs
    )

    release_build_task = decision.find_or_create_task(
        route_bucket="build.linux_x86-64_release",
        route_key=os.environ["GIT_SHA"],
        route_expiry=build_artifacts_expiry,

        task_name="Linux x86_64: release build",
        script="""
            ./mach build --release --with-debug-assertions -p servo
            ./etc/ci/lockfile_changed.sh
            tar -czf /target.tar.gz \
                target/release/servo \
                target/release/build/osmesa-src-*/output \
                target/release/build/osmesa-src-*/out/lib/gallium
        """,
        artifacts=[
            "/target.tar.gz",
        ],
        **build_kwargs
    )

    def create_run_task(*, script, env=None, **kwargs):
        fetch_build = """
            curl \
                "https://queue.taskcluster.net/v1/task/${BUILD_TASK_ID}/artifacts/public/target.tar.gz" \
                --retry 5 \
                --connect-timeout 10 \
                --location \
                --fail \
                | tar -xz
        """
        decision.create_task(
            script=fetch_build + script,
            env=dict(**env or {}, BUILD_TASK_ID=release_build_task),
            dependencies=[release_build_task],
            max_run_time_minutes=60,
            dockerfile="run-x86_64-linux.dockerfile",
            **kwargs
        )

    total_chunks = 4
    for i in range(total_chunks):
        chunk = i + 1
        create_run_task(
            task_name="Linux x86_64: WPT chunk %s / %s" % (chunk, total_chunks),
            script="""
                ./mach test-wpt \
                    --release \
                    --processes 24 \
                    --total-chunks "$TOTAL_CHUNKS" \
                    --this-chunk "$THIS_CHUNK" \
                    --log-raw test-wpt.log \
                    --log-errorsummary wpt-errorsummary.log \
                    --always-succeed
                ./mach filter-intermittents\
                    wpt-errorsummary.log \
                    --log-intermittents intermittents.log \
                    --log-filteredsummary filtered-wpt-errorsummary.log \
                    --tracker-api default
            """,
            # FIXME: --reporter-api default
            # IndexError: list index out of range
            # File "/repo/python/servo/testing_commands.py", line 533, in filter_intermittents
            #   pull_request = int(last_merge.split(' ')[4][1:])
            env={
                "TOTAL_CHUNKS": total_chunks,
                "THIS_CHUNK": chunk,
            },
        )

    create_run_task(
        task_name="Linux x86_64: WPT extra",
        script="""
            ./mach test-wpt-failure
            ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 \
                --log-raw test-wpt-mp.log \
                --log-errorsummary wpt-mp-errorsummary.log \
                eventsource
        """,
    )

if __name__ == "__main__":
    os.chdir(os.path.join(".", os.path.dirname(__file__)))
    main()
<commit_msg>Keep log files<commit_after># coding: utf8

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import os.path
from decisionlib import DecisionTask

def main():
    decision = DecisionTask(
        project_name="Servo",  # Used in task names
        route_prefix="project.servo.servo",
        worker_type="servo-docker-worker",
    )

    # FIXME: remove this before merging in servo/servo
    os.environ["GIT_URL"] = "https://github.com/SimonSapin/servo"
    os.environ["GIT_REF"] = "refs/heads/taskcluster-experiments-20180920"
    os.environ["GIT_SHA"] = "a6dbfdd29f9b3f0ce0c13adc79fad99538a9a44b"
    decision.docker_image_cache_expiry = "1 week"
    decision.route_prefix = "project.servo.servo-taskcluster-experiments"
    # ~

    build_artifacts_expiry = "1 week"
    log_artifacts_expiry = "1 year"

    # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/caches
    cache_scopes = [
        "docker-worker:cache:cargo-*",
    ]
    build_caches = {
        "cargo-registry-cache": "/root/.cargo/registry",
        "cargo-git-cache": "/root/.cargo/git",
        "cargo-rustup": "/root/.rustup",
        "cargo-sccache": "/root/.cache/sccache",
    }
    build_env = {
        "RUST_BACKTRACE": "1",
        "RUSTFLAGS": "-Dwarnings",
        "CARGO_INCREMENTAL": "0",
        "SCCACHE_IDLE_TIMEOUT": "1200",
        "CCACHE": "sccache",
        "RUSTC_WRAPPER": "sccache",
    }
    build_kwargs = {
        "max_run_time_minutes": 60,
        "dockerfile": "build-x86_64-linux.dockerfile",
        "env": build_env,
        "scopes": cache_scopes,
        "cache": build_caches,
    }

    decision.create_task(
        task_name="Linux x86_64: tidy + dev build + unit tests",
        script="""
            ./mach test-tidy --no-progress --all
            ./mach build --dev
            ./mach test-unit
            ./mach package --dev
            ./mach test-tidy --no-progress --self-test
            python ./etc/memory_reports_over_time.py --test
            ./etc/ci/lockfile_changed.sh
            ./etc/ci/check_no_panic.sh
        """,
        **build_kwargs
    )

    release_build_task = decision.find_or_create_task(
        route_bucket="build.linux_x86-64_release",
        route_key=os.environ["GIT_SHA"],
        route_expiry=build_artifacts_expiry,

        task_name="Linux x86_64: release build",
        script="""
            ./mach build --release --with-debug-assertions -p servo
            ./etc/ci/lockfile_changed.sh
            tar -czf /target.tar.gz \
                target/release/servo \
                target/release/build/osmesa-src-*/output \
                target/release/build/osmesa-src-*/out/lib/gallium
        """,
        artifacts=[
            "/target.tar.gz",
        ],
        **build_kwargs
    )

    def create_run_task(*, script, env=None, **kwargs):
        fetch_build = """
            curl \
                "https://queue.taskcluster.net/v1/task/${BUILD_TASK_ID}/artifacts/public/target.tar.gz" \
                --retry 5 \
                --connect-timeout 10 \
                --location \
                --fail \
                | tar -xz
        """
        kwargs.setdefault("artifacts", []).extend(
            ("/repo/" + word, log_artifacts_expiry)
            for word in script.split() if word.endswith(".log")
        )
        decision.create_task(
            script=fetch_build + script,
            env=dict(**env or {}, BUILD_TASK_ID=release_build_task),
            dependencies=[release_build_task],
            max_run_time_minutes=60,
            dockerfile="run-x86_64-linux.dockerfile",
            **kwargs
        )

    total_chunks = 4
    for i in range(total_chunks):
        chunk = i + 1
        create_run_task(
            task_name="Linux x86_64: WPT chunk %s / %s" % (chunk, total_chunks),
            script="""
                ./mach test-wpt \
                    --release \
                    --processes 24 \
                    --total-chunks "$TOTAL_CHUNKS" \
                    --this-chunk "$THIS_CHUNK" \
                    --log-raw test-wpt.log \
                    --log-errorsummary wpt-errorsummary.log \
                    --always-succeed
                ./mach filter-intermittents\
                    wpt-errorsummary.log \
                    --log-intermittents intermittents.log \
                    --log-filteredsummary filtered-wpt-errorsummary.log \
                    --tracker-api default
            """,
            # FIXME: --reporter-api default
            # IndexError: list index out of range
            # File "/repo/python/servo/testing_commands.py", line 533, in filter_intermittents
            #   pull_request = int(last_merge.split(' ')[4][1:])
            env={
                "TOTAL_CHUNKS": total_chunks,
                "THIS_CHUNK": chunk,
            },
        )

    create_run_task(
        task_name="Linux x86_64: WPT extra",
        script="""
            ./mach test-wpt-failure
            ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 \
                --log-raw test-wpt-mp.log \
                --log-errorsummary wpt-mp-errorsummary.log \
                eventsource
        """,
    )

if __name__ == "__main__":
    os.chdir(os.path.join(".", os.path.dirname(__file__)))
    main()
<|endoftext|>


<commit_before>"""
Transformations that primarily involve numerical computation on variables.
"""
import math
import numpy as np
import pandas as pd
from bids.utils import listify
from .base import Transformation
from bids.analysis import hrf
from bids.variables import SparseRunVariable,  DenseRunVariable

def _fractional_gcd(vals, res=0.001):
    from functools import reduce
    from math import gcd
    return reduce(gcd, (int(np.round(val / res)) for val in vals)) * res

class Convolve(Transformation):
    """Convolve the input variable with an HRF.

    Parameters
    ----------
    var : Variable
        The variable to convolve.
    model : str
        The name of the HRF model to apply. Must be one of 'spm',
        'glover', or 'fir'.
    derivative : bool
        Whether or not to include the temporal derivative.
    dispersion : bool
        Whether or not to include the dispersion derivative.
    fir_delays : iterable
        A list or iterable of delays to use if model is
        'fir' (ignored otherwise). Spacing between delays must be fixed.

    Notes
    -----
    Uses the HRF convolution functions implemented in nistats.
    """

    _input_type = 'variable'
    _return_type = 'variable'

    def _transform(self, var, model='spm', derivative=False, dispersion=False,
                   fir_delays=None):

        model = model.lower()

        df = var.to_df(entities=False)

        if isinstance(var, SparseRunVariable):
            sampling_rate = self.collection.sampling_rate
            dur = var.get_duration()
            resample_frames = np.linspace(
                0, dur, int(math.ceil(dur * sampling_rate)), endpoint=False)
            safety = 2  # Double frequency to resolve events
        else:
            resample_frames = df['onset'].values
            sampling_rate = var.sampling_rate
            safety = 1  # Maximum signal resolution is already 0.5 * SR

        vals = df[['onset', 'duration', 'amplitude']].values.T

        if model in ['spm', 'glover']:
            if derivative:
                model += ' + derivative'
            if dispersion:
                model += ' + dispersion'
        elif model != 'fir':
            raise ValueError("Model must be one of 'spm', 'glover', or 'fir'.")

        # Sampling at >100Hz will never be useful, but can be wildly expensive
        max_freq, min_interval = 100, 0.01
        # Sampling at <1Hz can degrade signals
        min_freq, max_interval = 1, 1

        # Given the sampling rate, determine an oversampling factor to ensure that
        # events can be modeled with reasonable precision
        unique_onsets = np.unique(df.onset)
        unique_durations = np.unique(df.duration)
        # Align existing data ticks with, event onsets and offsets, up to ms resolution
        # Note that GCD ignores zeros, so 0 onsets and impulse responses (0 durations) do
        # not harm this.
        required_resolution = _fractional_gcd(
            np.concatenate((unique_onsets, unique_durations)),
            res=min_interval)
        # Bound the effective sampling rate between min_freq and max_freq
        effective_sr = max(min_freq, min(safety / required_resolution, max_freq))
        convolved = hrf.compute_regressor(
            vals, model, resample_frames, fir_delays=fir_delays, min_onset=0,
            oversampling=np.ceil(effective_sr / sampling_rate)
            )

        return DenseRunVariable(
            name=var.name, values=convolved[0], run_info=var.run_info,
            source=var.source, sampling_rate=sampling_rate)

class Demean(Transformation):

    def _transform(self, data):
        return data - data.mean()

class Orthogonalize(Transformation):

    _variables_used = ('variables', 'other')
    _densify = ('variables', 'other')
    _align = ('other')

    def _transform(self, var, other):

        other = listify(other)

        # Set up X matrix and slice into it based on target variable indices
        X = np.array([self._variables[c].values.values.squeeze()
                      for c in other]).T
        X = X[var.index, :]
        assert len(X) == len(var)
        y = var.values
        _aX = np.c_[np.ones(len(y)), X]
        coefs, resids, rank, s = np.linalg.lstsq(_aX, y, rcond=None)
        result = pd.DataFrame(y - X.dot(coefs[1:]), index=var.index)
        return result

class Product(Transformation):

    _loopable = False
    _groupable = False
    _align = True
    _output_required = True

    def _transform(self, data):
        data = pd.concat(data, axis=1, sort=True)
        return data.product(1)

class Scale(Transformation):
    """Scale a variable.

    Parameters
    ----------
    data : :obj:`pandas.Series` or :obj:`pandas.DataFrame`
        The variables to scale.
    demean : bool
        If True, demean each column.
    rescale : bool
        If True, divide variables by their standard deviation.
    replace_na : str
        Whether/when to replace missing values with 0. If
        None, no replacement is performed. If 'before', missing values are
        replaced with 0's before scaling. If 'after', missing values are
        replaced with 0 after scaling.

    Notes
    -----
    If a constant column is passed in, and replace_na is None or 'before', an
    exception will be raised.
    """

    def _transform(self, data, demean=True, rescale=True, replace_na=None):
        if data.nunique() == 1 and replace_na in {None, 'before'}:
            val = data.unique()[0]
            raise ValueError("Cannot scale a column with constant value ({})! "
                             "If you want a constant column of 0's returned, "
                             "set replace_na to 'after'.".format(val))
        if replace_na == 'before':
            data = data.fillna(0.)
        if demean:
            data -= data.mean()
        if rescale:
            data /= data.std()
        if replace_na == 'after':
            data = data.fillna(0.)
        return data

class Sum(Transformation):

    _loopable = False
    _groupable = False
    _align = True
    _output_required = True

    def _transform(self, data, weights=None):
        data = pd.concat(data, axis=1, sort=True)
        if weights is None:
            weights = np.ones(data.shape[1])
        else:
            weights = np.array(weights)
            if len(weights.ravel()) != data.shape[1]:
                raise ValueError("If weights are passed to sum(), the number "
                                 "of elements must equal number of variables"
                                 " being summed.")
        return (data * weights).sum(axis=1)

class Threshold(Transformation):
    """Threshold and/or binarize a variable.

    Parameters
    ----------
    data :obj:`pandas.Series` or :obj:`pandas.DataFrame`
        The pandas structure to threshold.
    threshold : float
        The value to binarize around (values above will
        be assigned 1, values below will be assigned 0).
    binarize : bool
        If True, binarizes all non-zero values (i.e., every
        non-zero value will be set to 1).
    above : bool
        Specifies which values to retain with respect to the
        cut-off. If True, all value above the threshold will be kept; if
        False, all values below the threshold will be kept. Defaults to
        True.
    signed : bool
        Specifies whether to treat the threshold as signed
        (default) or unsigned. For example, when passing above=True and
        threshold=3, if signed=True, all and only values above +3 would be
        retained. If signed=False, all absolute values > 3 would be retained
        (i.e.,values in  the range -3 < X < 3 would be set to 0).
    """

    _groupable = False

    def _transform(self, data, threshold=0., binarize=False, above=True,
                   signed=True):
        if not signed:
            threshold = np.abs(threshold)
            data = data.abs()
        keep = data >= threshold if above else data <= threshold
        data[~keep] = 0
        if binarize:
            data[keep] = 1
        return data

class And_(Transformation):
    """Logical AND on two or more variables.

    Parameters
    ----------
    dfs : list of :obj:`pandas.DataFrame`
        variables to enter into the conjunction.
    """

    _loopable = False
    _groupable = False
    _output_required = True

    def _transform(self, dfs):
        df = pd.concat(dfs, axis=1, sort=True)
        return df.all(axis=1).astype(int)

class Not(Transformation):
    """Logical negation of a variable.

    Parameters
    ----------
    var : :obj:`pandas.Series`
        Variable to negate. Must be convertible to bool.
    """

    _loopable = True
    _groupable = False

    def _transform(self, var):
        return ~var.astype(bool)

class Or_(Transformation):
    """Logical OR (inclusive) on two or more variables.

    Parameters
    ----------
    dfs : list of :obj:`pandas.DataFrame`
        variables to enter into the disjunction.
    """

    _loopable = False
    _groupable = False
    _output_required = True

    def _transform(self, dfs):
        df = pd.concat(dfs, axis=1, sort=True)
        return df.any(axis=1).astype(int)
<commit_msg>Logical operations require aligned input<commit_after>"""
Transformations that primarily involve numerical computation on variables.
"""
import math
import numpy as np
import pandas as pd
from bids.utils import listify
from .base import Transformation
from bids.analysis import hrf
from bids.variables import SparseRunVariable,  DenseRunVariable

def _fractional_gcd(vals, res=0.001):
    from functools import reduce
    from math import gcd
    return reduce(gcd, (int(np.round(val / res)) for val in vals)) * res

class Convolve(Transformation):
    """Convolve the input variable with an HRF.

    Parameters
    ----------
    var : Variable
        The variable to convolve.
    model : str
        The name of the HRF model to apply. Must be one of 'spm',
        'glover', or 'fir'.
    derivative : bool
        Whether or not to include the temporal derivative.
    dispersion : bool
        Whether or not to include the dispersion derivative.
    fir_delays : iterable
        A list or iterable of delays to use if model is
        'fir' (ignored otherwise). Spacing between delays must be fixed.

    Notes
    -----
    Uses the HRF convolution functions implemented in nistats.
    """

    _input_type = 'variable'
    _return_type = 'variable'

    def _transform(self, var, model='spm', derivative=False, dispersion=False,
                   fir_delays=None):

        model = model.lower()

        df = var.to_df(entities=False)

        if isinstance(var, SparseRunVariable):
            sampling_rate = self.collection.sampling_rate
            dur = var.get_duration()
            resample_frames = np.linspace(
                0, dur, int(math.ceil(dur * sampling_rate)), endpoint=False)
            safety = 2  # Double frequency to resolve events
        else:
            resample_frames = df['onset'].values
            sampling_rate = var.sampling_rate
            safety = 1  # Maximum signal resolution is already 0.5 * SR

        vals = df[['onset', 'duration', 'amplitude']].values.T

        if model in ['spm', 'glover']:
            if derivative:
                model += ' + derivative'
            if dispersion:
                model += ' + dispersion'
        elif model != 'fir':
            raise ValueError("Model must be one of 'spm', 'glover', or 'fir'.")

        # Sampling at >100Hz will never be useful, but can be wildly expensive
        max_freq, min_interval = 100, 0.01
        # Sampling at <1Hz can degrade signals
        min_freq, max_interval = 1, 1

        # Given the sampling rate, determine an oversampling factor to ensure that
        # events can be modeled with reasonable precision
        unique_onsets = np.unique(df.onset)
        unique_durations = np.unique(df.duration)
        # Align existing data ticks with, event onsets and offsets, up to ms resolution
        # Note that GCD ignores zeros, so 0 onsets and impulse responses (0 durations) do
        # not harm this.
        required_resolution = _fractional_gcd(
            np.concatenate((unique_onsets, unique_durations)),
            res=min_interval)
        # Bound the effective sampling rate between min_freq and max_freq
        effective_sr = max(min_freq, min(safety / required_resolution, max_freq))
        convolved = hrf.compute_regressor(
            vals, model, resample_frames, fir_delays=fir_delays, min_onset=0,
            oversampling=np.ceil(effective_sr / sampling_rate)
            )

        return DenseRunVariable(
            name=var.name, values=convolved[0], run_info=var.run_info,
            source=var.source, sampling_rate=sampling_rate)

class Demean(Transformation):

    def _transform(self, data):
        return data - data.mean()

class Orthogonalize(Transformation):

    _variables_used = ('variables', 'other')
    _densify = ('variables', 'other')
    _align = ('other')

    def _transform(self, var, other):

        other = listify(other)

        # Set up X matrix and slice into it based on target variable indices
        X = np.array([self._variables[c].values.values.squeeze()
                      for c in other]).T
        X = X[var.index, :]
        assert len(X) == len(var)
        y = var.values
        _aX = np.c_[np.ones(len(y)), X]
        coefs, resids, rank, s = np.linalg.lstsq(_aX, y, rcond=None)
        result = pd.DataFrame(y - X.dot(coefs[1:]), index=var.index)
        return result

class Product(Transformation):

    _loopable = False
    _groupable = False
    _align = True
    _output_required = True

    def _transform(self, data):
        data = pd.concat(data, axis=1, sort=True)
        return data.product(1)

class Scale(Transformation):
    """Scale a variable.

    Parameters
    ----------
    data : :obj:`pandas.Series` or :obj:`pandas.DataFrame`
        The variables to scale.
    demean : bool
        If True, demean each column.
    rescale : bool
        If True, divide variables by their standard deviation.
    replace_na : str
        Whether/when to replace missing values with 0. If
        None, no replacement is performed. If 'before', missing values are
        replaced with 0's before scaling. If 'after', missing values are
        replaced with 0 after scaling.

    Notes
    -----
    If a constant column is passed in, and replace_na is None or 'before', an
    exception will be raised.
    """

    def _transform(self, data, demean=True, rescale=True, replace_na=None):
        if data.nunique() == 1 and replace_na in {None, 'before'}:
            val = data.unique()[0]
            raise ValueError("Cannot scale a column with constant value ({})! "
                             "If you want a constant column of 0's returned, "
                             "set replace_na to 'after'.".format(val))
        if replace_na == 'before':
            data = data.fillna(0.)
        if demean:
            data -= data.mean()
        if rescale:
            data /= data.std()
        if replace_na == 'after':
            data = data.fillna(0.)
        return data

class Sum(Transformation):

    _loopable = False
    _groupable = False
    _align = True
    _output_required = True

    def _transform(self, data, weights=None):
        data = pd.concat(data, axis=1, sort=True)
        if weights is None:
            weights = np.ones(data.shape[1])
        else:
            weights = np.array(weights)
            if len(weights.ravel()) != data.shape[1]:
                raise ValueError("If weights are passed to sum(), the number "
                                 "of elements must equal number of variables"
                                 " being summed.")
        return (data * weights).sum(axis=1)

class Threshold(Transformation):
    """Threshold and/or binarize a variable.

    Parameters
    ----------
    data :obj:`pandas.Series` or :obj:`pandas.DataFrame`
        The pandas structure to threshold.
    threshold : float
        The value to binarize around (values above will
        be assigned 1, values below will be assigned 0).
    binarize : bool
        If True, binarizes all non-zero values (i.e., every
        non-zero value will be set to 1).
    above : bool
        Specifies which values to retain with respect to the
        cut-off. If True, all value above the threshold will be kept; if
        False, all values below the threshold will be kept. Defaults to
        True.
    signed : bool
        Specifies whether to treat the threshold as signed
        (default) or unsigned. For example, when passing above=True and
        threshold=3, if signed=True, all and only values above +3 would be
        retained. If signed=False, all absolute values > 3 would be retained
        (i.e.,values in  the range -3 < X < 3 would be set to 0).
    """

    _groupable = False

    def _transform(self, data, threshold=0., binarize=False, above=True,
                   signed=True):
        if not signed:
            threshold = np.abs(threshold)
            data = data.abs()
        keep = data >= threshold if above else data <= threshold
        data[~keep] = 0
        if binarize:
            data[keep] = 1
        return data

class And_(Transformation):
    """Logical AND on two or more variables.

    Parameters
    ----------
    dfs : list of :obj:`pandas.DataFrame`
        variables to enter into the conjunction.
    """

    _loopable = False
    _groupable = False
    _output_required = True
    _align = True

    def _transform(self, dfs):
        df = pd.concat(dfs, axis=1, sort=True)
        return df.all(axis=1).astype(int)

class Not(Transformation):
    """Logical negation of a variable.

    Parameters
    ----------
    var : :obj:`pandas.Series`
        Variable to negate. Must be convertible to bool.
    """

    _loopable = True
    _groupable = False
    _align = True

    def _transform(self, var):
        return ~var.astype(bool)

class Or_(Transformation):
    """Logical OR (inclusive) on two or more variables.

    Parameters
    ----------
    dfs : list of :obj:`pandas.DataFrame`
        variables to enter into the disjunction.
    """

    _loopable = False
    _groupable = False
    _output_required = True
    _align = True

    def _transform(self, dfs):
        df = pd.concat(dfs, axis=1, sort=True)
        return df.any(axis=1).astype(int)
<|endoftext|>


<commit_before>from collections import defaultdict
from couchdbkit import ResourceNotFound
from bihar.reports.indicators.filters import A_MONTH, is_pregnant_mother, get_add, get_edd
from couchforms.safe_index import safe_index
from dimagi.utils.parsing import string_to_datetime
import datetime as dt
from bihar.reports.indicators.visits import visit_is, get_related_prop

EMPTY = (0,0)
GRACE_PERIOD = dt.timedelta(days=7)

def _num_denom(num, denom):
    return "%s/%s" % (num, denom)

def _in_last_month(date):
    today = dt.datetime.today().date()
    return today - A_MONTH < date < today

def _in_timeframe(date, days):
    today = dt.datetime.today().date()
    return today - dt.timedelta(days=days) < date < today

def _mother_due_in_window(case, days):
    get_visitduedate = lambda case: case.edd - dt.timedelta(days=days) + GRACE_PERIOD
    return is_pregnant_mother(case) and get_edd(case) and _in_last_month(get_visitduedate(case))

def _mother_delivered_in_window(case, days):
    get_visitduedate = lambda case: case.add + dt.timedelta(days=days) + GRACE_PERIOD
    return is_pregnant_mother(case) and get_add(case) and _in_last_month(get_visitduedate(case))

def _num_denom_count(cases, num_func, denom_func):
    num = denom = 0
    for case in cases:
        denom_diff = denom_func(case)
        if denom_diff:
            denom += denom_diff
            num_diff = num_func(case)
            assert num_diff <= denom_diff
            # this is to prevent the numerator from ever passing the denominator
            # though is probably not totally accurate
            num += num_diff
    return _num_denom(num, denom)

def _visits_due(case, schedule):
    return [i + 1 for i, days in enumerate(schedule) \
            if _mother_delivered_in_window(case, days)]

def _visits_done(case, schedule, type):
    due = _visits_due(case, schedule)
    count = len(filter(lambda a: visit_is(a, type), case.actions))
    return len([v for v in due if count > v])

def _delivered_in_timeframe(case, days):
    return is_pregnant_mother(case) and get_add(case) and _in_timeframe(case.add, days)

def _delivered_at_in_timeframe(case, at, days):
    at = at if isinstance(at, list) else [at]
    return getattr(case, 'birth_place', None) in at and _delivered_in_timeframe(case, days)

def _get_time_of_visit_after_birth(case):
    for action in case.actions:
        if action.updated_unknown_properties.get("add", None):
            return action.date
    return None

def _visited_in_timeframe_of_birth(case, days):
    visit_time = _get_time_of_visit_after_birth(case)
    time_birth = get_related_prop(case, "time_of_birth") or get_add(case) # use add if time_of_birth can't be found
    if visit_time and time_birth:
        if isinstance(time_birth, dt.date):
            time_birth = dt.datetime.combine(time_birth, dt.datetime.time(dt.datetime.now())) #convert date to dt.datetime
        return time_birth < visit_time < time_birth + dt.timedelta(days=days)
    return False

def _weak_babies(case): # :(
    return is_pregnant_mother(case) and\
           (getattr(case, 'recently_delivered', None) == "yes" or get_related_prop(case, 'birth_status') == "live_birth")

# NOTE: cases in, values out might not be the right API
# but it's what we need for the first set of stuff.
# might want to revisit.

# NOTE: this is going to be slooooow
def bp2_last_month(cases):
    due = lambda case: 1 if _mother_due_in_window(case, 75) else 0
    # make sure they've done 2 bp visits
    done = lambda case: 1 if len(filter(lambda a: visit_is(a, 'bp'), case.actions)) >= 2 else 0
    return _num_denom_count(cases, due, done)

def bp3_last_month(cases):
    due = lambda case: 1 if _mother_due_in_window(case, 45) else 0
    # make sure they've done 2 bp visits
    done = lambda case: 1 if len(filter(lambda a: visit_is(a, 'bp'), case.actions)) >= 3 else 0
    return _num_denom_count(cases, due, done)

def pnc_last_month(cases):
    pnc_schedule = (1, 3, 6)
    due = lambda case: len(_visits_due(case, pnc_schedule))
    done = lambda case: _visits_done(case, pnc_schedule, "pnc")
    return _num_denom_count(cases, done, due)

def eb_last_month(cases):
    eb_schedule = (14, 28, 60, 90, 120, 150)
    due = lambda case: len(_visits_due(case, eb_schedule))
    done = lambda case: _visits_done(case, eb_schedule, "eb")
    return _num_denom_count(cases, done, due)

def cf_last_month(cases):
    cf_schedule_in_months = (6, 7, 8, 9, 12, 15, 18)
    cf_schedule = (m * 30 for m in cf_schedule_in_months)
    due = lambda case: len(_visits_due(case, cf_schedule))
    done = lambda case: _visits_done(case, cf_schedule, "cf")
    return _num_denom_count(cases, done, due)

def hd_day(cases):
    valid_cases = filter(lambda case: _delivered_at_in_timeframe(case, 'home', 30), cases)
    denom = len(valid_cases)
    num = len(filter(lambda case:_visited_in_timeframe_of_birth(case, 1) , valid_cases))
    return _num_denom(num, denom)

def id_day(cases):
    valid_cases = filter(lambda case: _delivered_at_in_timeframe(case, ['private', 'public'], 30), cases)
    denom = len(valid_cases)
    num = len(filter(lambda case:_visited_in_timeframe_of_birth(case, 1) , valid_cases))
    return _num_denom(num, denom)

def idnb(cases):
    valid_cases = filter(lambda case: _delivered_at_in_timeframe(case, ['private', 'public'], 30) and
                                      get_related_prop(case, 'birth_status') == "live_birth", cases)
    denom = len(valid_cases)

    def breastfed_hour(case):
        dtf = get_related_prop(case, 'date_time_feed')
        tob = get_related_prop(case, 'time_of_birth')
        if dtf and tob:
            return dtf - tob <= dt.timedelta(hours=1)
        return False

    num = len(filter(lambda case: breastfed_hour(case), valid_cases))
    return _num_denom(num, denom)

def ptlb(cases, num_only=False):
    valid_cases = filter(lambda case: _weak_babies(case), cases)
    denom = len(valid_cases)
    num = len(filter(lambda case: getattr(case, 'term', None) == "pre_term", valid_cases))
    return _num_denom(num, denom) if not num_only else num

def lt2kglb(cases, num_only=False):
    valid_cases = filter(lambda case: _weak_babies(case), cases)
    denom = len(valid_cases)

    def over2(case):
        w = get_related_prop(case, 'weight')
        fw = get_related_prop(case, 'first_weight')
        return (w is not None and w < 2.0) or (fw is not None and fw < 2.0)

    num = len(filter(lambda case: over2(case), valid_cases))
    return _num_denom(num, denom) if not num_only else num

def _get_time_of_birth(form):
    try:
        time_of_birth = form.xpath('form/data/child_info/case/update/time_of_birth')
        assert time_of_birth is not None
    except AssertionError:
        time_of_birth = safe_index(
            form.xpath('form/data/child_info')[0],
            'case/update/time_of_birth'.split('/')
        )
    return time_of_birth

def complications(cases, days, now=None):
    """
    DENOM: [
        any DELIVERY forms with (
            /data/complications = 'yes'
        ) in last 30 days
        PLUS any PNC forms with ( # 'any applicable from PNC forms with' (?)
            /data/abdominal_pain ='yes' or
            /data/bleeding = 'yes' or
            /data/discharge = 'yes' or
            /data/fever = 'yes' or
            /data/pain_urination = 'yes'
        ) in the last 30 days
        PLUS any REGISTRATION forms with (
            /data/abd_pain ='yes' or    # == abdominal_pain
            /data/fever = 'yes' or
            /data/pain_urine = 'yes' or    # == pain_urination
            /data/vaginal_discharge = 'yes'    # == discharge
        ) with /data/add in last 30 days
        PLUS any EBF forms with (
            /data/abdominal_pain ='yes' or
            /data/bleeding = 'yes' or
            /data/discharge = 'yes' or
            /data/fever = 'yes' or
            /data/pain_urination = 'yes'
        ) in last 30 days    # note, don't exist in EBF yet, but will shortly
    ]
    NUM: [
        filter (
            DELIVERY ? form.meta.timeStart - /data/child_info/case/update/time_of_birth,
            REGISTRATION|PNC|EBF ? form.meta.timeStart - case.add
        ) < `days` days
    ]
    """
    #https://bitbucket.org/dimagi/cc-apps/src/caab8f93c1e48d702b5d9032ef16c9cec48868f0/bihar/mockup/bihar_pnc.xml
    #https://bitbucket.org/dimagi/cc-apps/src/caab8f93c1e48d702b5d9032ef16c9cec48868f0/bihar/mockup/bihar_del.xml
    #https://bitbucket.org/dimagi/cc-apps/src/caab8f93c1e48d702b5d9032ef16c9cec48868f0/bihar/mockup/bihar_registration.xml
    #https://bitbucket.org/dimagi/cc-apps/src/caab8f93c1e48d702b5d9032ef16c9cec48868f0/bihar/mockup/bihar_ebf.xml
    now = now or dt.datetime.utcnow()
    PNC = 'http://bihar.commcarehq.org/pregnancy/pnc'
    DELIVERY = 'http://bihar.commcarehq.org/pregnancy/del'
    REGISTRATION = 'http://bihar.commcarehq.org/pregnancy/registration'
    EBF = 'https://bitbucket.org/dimagi/cc-apps/src/caab8f93c1e48d702b5d9032ef16c9cec48868f0/bihar/mockup/bihar_ebf.xml'
    _pnc_ebc_complications = [
        '/data/abdominal_pain',
        '/data/bleeding',
        '/data/discharge',
        '/data/fever',
        '/data/pain_urination',
        ]
    complications_by_form = {
        DELIVERY: [
            '/data/complications'
        ],
        PNC: _pnc_ebc_complications,
        EBF: _pnc_ebc_complications,
        REGISTRATION: [
            '/data/abd_pain',
            '/data/fever',
            '/data/pain_urine',
            '/data/vaginal_discharge',
            ],
        }

    debug = defaultdict(int)
    def get_forms(case, days=30):
        for action in case.actions:
            debug['forms_scanned'] +=1
            if now - dt.timedelta(days=days) <= action.date <= now:
                debug['submitted_in_time'] +=1
                try:
                    yield action.xform
                except ResourceNotFound:
                    debug['bad_xform_refs'] += 1

    done = 0
    due = 0
    days = dt.timedelta(days=days)
    for case in cases:
        for form in get_forms(case):
            try:
                complication_paths = complications_by_form[form.xmlns]
            except KeyError:
                continue
            debug['relevent_xmlns'] += 1
            for p in complication_paths:
                has_complications = form.xpath('form/data/complications')
                if has_complications == 'no':
                    debug['%s complications is no' % form.xmlns.split('/')[-1]] += 1
                elif has_complications is None:
                    debug['%s complications dne' % form.xmlns.split('/')[-1]] += 1
                else:
                    debug['has_complications'] += 1
                print form.get_id, p, form.xpath('form' + p)
                if form.xpath('form' + p) == 'yes':
                    due += 1
                    if form.xmlns == DELIVERY:
                        add = _get_time_of_birth(form)
                    else:
                        add = get_add(case)
                    add = string_to_datetime(add)
                    if form.metatdata.timeStart - add < days:
                        done += 1
    return "%s/%s,<br/>debug: %s" % (done, due, dict(debug))<commit_msg>fix calculation functions to work off of xforms<commit_after>from collections import defaultdict
from couchdbkit import ResourceNotFound
from bihar.reports.indicators.filters import A_MONTH, is_pregnant_mother, get_add, get_edd
from couchforms.safe_index import safe_index
from dimagi.utils.parsing import string_to_datetime
import datetime as dt
from bihar.reports.indicators.visits import visit_is, get_related_prop

EMPTY = (0,0)
GRACE_PERIOD = dt.timedelta(days=7)

def _num_denom(num, denom):
    return "%s/%s" % (num, denom)

def _in_last_month(date):
    today = dt.datetime.today().date()
    return today - A_MONTH < date < today

def _in_timeframe(date, days):
    today = dt.datetime.today().date()
    return today - dt.timedelta(days=days) < date < today

def _mother_due_in_window(case, days):
    get_visitduedate = lambda case: case.edd - dt.timedelta(days=days) + GRACE_PERIOD
    return is_pregnant_mother(case) and get_edd(case) and _in_last_month(get_visitduedate(case))

def _mother_delivered_in_window(case, days):
    get_visitduedate = lambda case: case.add + dt.timedelta(days=days) + GRACE_PERIOD
    return is_pregnant_mother(case) and get_add(case) and _in_last_month(get_visitduedate(case))

def _num_denom_count(cases, num_func, denom_func):
    num = denom = 0
    for case in cases:
        denom_diff = denom_func(case)
        if denom_diff:
            denom += denom_diff
            num_diff = num_func(case)
            assert num_diff <= denom_diff
            # this is to prevent the numerator from ever passing the denominator
            # though is probably not totally accurate
            num += num_diff
    return _num_denom(num, denom)

def _visits_due(case, schedule):
    return [i + 1 for i, days in enumerate(schedule) \
            if _mother_delivered_in_window(case, days)]

def _visits_done(case, schedule, type):
    due = _visits_due(case, schedule)
    count = len(filter(lambda a: visit_is(a, type), case.actions))
    return len([v for v in due if count > v])

def _delivered_in_timeframe(case, days):
    return is_pregnant_mother(case) and get_add(case) and _in_timeframe(case.add, days)

def _delivered_at_in_timeframe(case, at, days):
    at = at if isinstance(at, list) else [at]
    return getattr(case, 'birth_place', None) in at and _delivered_in_timeframe(case, days)

def _get_time_of_visit_after_birth(case):
    for action in case.actions:
        if action.updated_unknown_properties.get("add", None):
            return action.date
    return None

def _visited_in_timeframe_of_birth(case, days):
    visit_time = _get_time_of_visit_after_birth(case)
    time_birth = get_related_prop(case, "time_of_birth") or get_add(case) # use add if time_of_birth can't be found
    if visit_time and time_birth:
        if isinstance(time_birth, dt.date):
            time_birth = dt.datetime.combine(time_birth, dt.datetime.time(dt.datetime.now())) #convert date to dt.datetime
        return time_birth < visit_time < time_birth + dt.timedelta(days=days)
    return False

def _get_actions(case, action_filter=lambda a: True):
    for action in case.actions:
        if action_filter(action):
            yield action

def _get_forms(case, action_filter=lambda a: True, form_filter=lambda f: True):
    for action in _get_actions(case, action_filter):
        if getattr(action, 'xform', None) and form_filter(action.xform):
            yield action.xform

def _weak_babies(case, days=None): # :(
    def af(action):
        if not days:
            return True
        now = dt.datetime.now()
        return now - dt.timedelta(days=days) <= action.date <= now

    def recently_delivered(case):
        for form in _get_forms(case, action_filter=af):
            if form.xpath("form/data/recently_delivered") == 'yes':
                return True
        return False

    return is_pregnant_mother(case) and\
           (recently_delivered(case) or get_related_prop(case, 'birth_status') == "live_birth")

# NOTE: cases in, values out might not be the right API
# but it's what we need for the first set of stuff.
# might want to revisit.

# NOTE: this is going to be slooooow
def bp2_last_month(cases):
    due = lambda case: 1 if _mother_due_in_window(case, 75) else 0
    # make sure they've done 2 bp visits
    done = lambda case: 1 if len(filter(lambda a: visit_is(a, 'bp'), case.actions)) >= 2 else 0
    return _num_denom_count(cases, due, done)

def bp3_last_month(cases):
    due = lambda case: 1 if _mother_due_in_window(case, 45) else 0
    # make sure they've done 2 bp visits
    done = lambda case: 1 if len(filter(lambda a: visit_is(a, 'bp'), case.actions)) >= 3 else 0
    return _num_denom_count(cases, due, done)

def pnc_last_month(cases):
    pnc_schedule = (1, 3, 6)
    due = lambda case: len(_visits_due(case, pnc_schedule))
    done = lambda case: _visits_done(case, pnc_schedule, "pnc")
    return _num_denom_count(cases, done, due)

def eb_last_month(cases):
    eb_schedule = (14, 28, 60, 90, 120, 150)
    due = lambda case: len(_visits_due(case, eb_schedule))
    done = lambda case: _visits_done(case, eb_schedule, "eb")
    return _num_denom_count(cases, done, due)

def cf_last_month(cases):
    cf_schedule_in_months = (6, 7, 8, 9, 12, 15, 18)
    cf_schedule = (m * 30 for m in cf_schedule_in_months)
    due = lambda case: len(_visits_due(case, cf_schedule))
    done = lambda case: _visits_done(case, cf_schedule, "cf")
    return _num_denom_count(cases, done, due)

def hd_day(cases):
    valid_cases = filter(lambda case: _delivered_at_in_timeframe(case, 'home', 30), cases)
    denom = len(valid_cases)
    num = len(filter(lambda case:_visited_in_timeframe_of_birth(case, 1) , valid_cases))
    return _num_denom(num, denom)

def id_day(cases):
    valid_cases = filter(lambda case: _delivered_at_in_timeframe(case, ['private', 'public'], 30), cases)
    denom = len(valid_cases)
    num = len(filter(lambda case:_visited_in_timeframe_of_birth(case, 1) , valid_cases))
    return _num_denom(num, denom)

def idnb(cases):
    valid_cases = filter(lambda case: _delivered_at_in_timeframe(case, ['private', 'public'], 30) and
                                      get_related_prop(case, 'birth_status') == "live_birth", cases)
    denom = len(valid_cases)

    def breastfed_hour(case):
        dtf = get_related_prop(case, 'date_time_feed')
        tob = get_related_prop(case, 'time_of_birth')
        if dtf and tob:
            return dtf - tob <= dt.timedelta(hours=1)
        return False

    num = len(filter(lambda case: breastfed_hour(case), valid_cases))
    return _num_denom(num, denom)

def ptlb(cases, num_only=False):
    valid_cases = filter(lambda case: _weak_babies(case, 30), cases)
    denom = len(valid_cases)
    num = len(filter(lambda case: getattr(case, 'term', None) == "pre_term", valid_cases))
    return _num_denom(num, denom) if not num_only else num

def lt2kglb(cases, num_only=False):
    valid_cases = filter(lambda case: _weak_babies(case, 30), cases)
    denom = len(valid_cases)

    def over2(case):
        w = get_related_prop(case, 'weight')
        fw = get_related_prop(case, 'first_weight')
        return (w is not None and w < 2.0) or (fw is not None and fw < 2.0)

    num = len(filter(lambda case: over2(case), valid_cases))
    return _num_denom(num, denom) if not num_only else num

def _get_time_of_birth(form):
    try:
        time_of_birth = form.xpath('form/data/child_info/case/update/time_of_birth')
        assert time_of_birth is not None
    except AssertionError:
        time_of_birth = safe_index(
            form.xpath('form/data/child_info')[0],
            'case/update/time_of_birth'.split('/')
        )
    return time_of_birth

def complications(cases, days, now=None):
    """
    DENOM: [
        any DELIVERY forms with (
            /data/complications = 'yes'
        ) in last 30 days
        PLUS any PNC forms with ( # 'any applicable from PNC forms with' (?)
            /data/abdominal_pain ='yes' or
            /data/bleeding = 'yes' or
            /data/discharge = 'yes' or
            /data/fever = 'yes' or
            /data/pain_urination = 'yes'
        ) in the last 30 days
        PLUS any REGISTRATION forms with (
            /data/abd_pain ='yes' or    # == abdominal_pain
            /data/fever = 'yes' or
            /data/pain_urine = 'yes' or    # == pain_urination
            /data/vaginal_discharge = 'yes'    # == discharge
        ) with /data/add in last 30 days
        PLUS any EBF forms with (
            /data/abdominal_pain ='yes' or
            /data/bleeding = 'yes' or
            /data/discharge = 'yes' or
            /data/fever = 'yes' or
            /data/pain_urination = 'yes'
        ) in last 30 days    # note, don't exist in EBF yet, but will shortly
    ]
    NUM: [
        filter (
            DELIVERY ? form.meta.timeStart - /data/child_info/case/update/time_of_birth,
            REGISTRATION|PNC|EBF ? form.meta.timeStart - case.add
        ) < `days` days
    ]
    """
    #https://bitbucket.org/dimagi/cc-apps/src/caab8f93c1e48d702b5d9032ef16c9cec48868f0/bihar/mockup/bihar_pnc.xml
    #https://bitbucket.org/dimagi/cc-apps/src/caab8f93c1e48d702b5d9032ef16c9cec48868f0/bihar/mockup/bihar_del.xml
    #https://bitbucket.org/dimagi/cc-apps/src/caab8f93c1e48d702b5d9032ef16c9cec48868f0/bihar/mockup/bihar_registration.xml
    #https://bitbucket.org/dimagi/cc-apps/src/caab8f93c1e48d702b5d9032ef16c9cec48868f0/bihar/mockup/bihar_ebf.xml
    now = now or dt.datetime.utcnow()
    PNC = 'http://bihar.commcarehq.org/pregnancy/pnc'
    DELIVERY = 'http://bihar.commcarehq.org/pregnancy/del'
    REGISTRATION = 'http://bihar.commcarehq.org/pregnancy/registration'
    EBF = 'https://bitbucket.org/dimagi/cc-apps/src/caab8f93c1e48d702b5d9032ef16c9cec48868f0/bihar/mockup/bihar_ebf.xml'
    _pnc_ebc_complications = [
        '/data/abdominal_pain',
        '/data/bleeding',
        '/data/discharge',
        '/data/fever',
        '/data/pain_urination',
        ]
    complications_by_form = {
        DELIVERY: [
            '/data/complications'
        ],
        PNC: _pnc_ebc_complications,
        EBF: _pnc_ebc_complications,
        REGISTRATION: [
            '/data/abd_pain',
            '/data/fever',
            '/data/pain_urine',
            '/data/vaginal_discharge',
            ],
        }

    debug = defaultdict(int)
    def get_forms(case, days=30):
        for action in case.actions:
            debug['forms_scanned'] +=1
            if now - dt.timedelta(days=days) <= action.date <= now:
                debug['submitted_in_time'] +=1
                try:
                    yield action.xform
                except ResourceNotFound:
                    debug['bad_xform_refs'] += 1

    done = 0
    due = 0
    days = dt.timedelta(days=days)
    for case in cases:
        for form in get_forms(case):
            try:
                complication_paths = complications_by_form[form.xmlns]
            except KeyError:
                continue
            debug['relevent_xmlns'] += 1
            for p in complication_paths:
                has_complications = form.xpath('form/data/complications')
                if has_complications == 'no':
                    debug['%s complications is no' % form.xmlns.split('/')[-1]] += 1
                elif has_complications is None:
                    debug['%s complications dne' % form.xmlns.split('/')[-1]] += 1
                else:
                    debug['has_complications'] += 1
                print form.get_id, p, form.xpath('form' + p)
                if form.xpath('form' + p) == 'yes':
                    due += 1
                    if form.xmlns == DELIVERY:
                        add = _get_time_of_birth(form)
                    else:
                        add = get_add(case)
                    add = string_to_datetime(add)
                    if form.metatdata.timeStart - add < days:
                        done += 1
    return "%s/%s,<br/>debug: %s" % (done, due, dict(debug))<|endoftext|>


<commit_before>from billy.core import db, settings
from billy.importers.bills import elasticsearch_push
from billy.bin.commands import BaseCommand

import logging
logging.getLogger('boto').setLevel(logging.CRITICAL)
log = logging.getLogger('billy')

class ElasticsearchPush(BaseCommand):

    name = 'elasticsearch-push'
    help = 'sync bills to elasticsearch instance'

    def add_args(self):
        self.add_argument('abbrs', metavar='ABBR', type=str, nargs='+',
                          help='abbreviations for bills to update')

    def handle(self, args):
        for newpath in settings.SCRAPER_PATHS:
            sys.path.insert(0, newpath)
        for abbr in args.abbrs:
            for bill in db.bills.find({settings.LEVEL_FIELD: abbr.lower()},
                                      timeout=False):
                elasticsearch_push(bill)
<commit_msg>import sys in es push<commit_after>import sys

from billy.core import db, settings
from billy.importers.bills import elasticsearch_push
from billy.bin.commands import BaseCommand

import logging
logging.getLogger('boto').setLevel(logging.CRITICAL)
log = logging.getLogger('billy')

class ElasticsearchPush(BaseCommand):

    name = 'elasticsearch-push'
    help = 'sync bills to elasticsearch instance'

    def add_args(self):
        self.add_argument('abbrs', metavar='ABBR', type=str, nargs='+',
                          help='abbreviations for bills to update')

    def handle(self, args):
        for newpath in settings.SCRAPER_PATHS:
            sys.path.insert(0, newpath)
        for abbr in args.abbrs:
            for bill in db.bills.find({settings.LEVEL_FIELD: abbr.lower()},
                                      timeout=False):
                elasticsearch_push(bill)
<|endoftext|>


<commit_before>"""a non-blocking, non-threaded non-multiprocessing circuits web server"""

import rexviewer as r

try:
    import circuits
except ImportError: #not running within the viewer, but testing outside it
    import sys
    sys.path.append('..')

from circuits.web import Controller, Server, Static
import datetime
import os.path

PORT = 8000

#doesn't work in the embedded context. what would help?
#PATH = os.path.dirname(__file__) + "/screenshot/"
PATH = "C:\\work\\rex\\trunk\\bin\\pymodules\\webserver" + "/screenshot/"

IMAGEPATH = "C:/work/rex/trunk/bin/pymodules/webserver/screenshot/"

print "WEB PATH:", PATH

class WebServer(Server):
    """this is the component that autoload registers to the viewer"""
    def __init__(self):
        Server.__init__(self, PORT)
        self + WebController() + Static(docroot=PATH)

class WebController(Controller):
    def index(self):
        return "Hello World!"

    def render(self):
        timestr = datetime.datetime.today().isoformat()
        imgsrc = "image.png"
        r.takeScreenshot(IMAGEPATH, imgsrc)

        return """

Realxtend Naali viewer

at %s

        """ % (timestr, imgsrc)

#~ if __name__ == '__main__':
    #~ while 1:
        #~ print ".",
<commit_msg>twid: no other path is needed here, just the image path<commit_after>"""a non-blocking, non-threaded non-multiprocessing circuits web server"""

import rexviewer as r

try:
    import circuits
except ImportError: #not running within the viewer, but testing outside it
    import sys
    sys.path.append('..')

from circuits.web import Controller, Server, Static
import datetime
import os.path

PORT = 8000

#doesn't work in the embedded context. what would help?
#PATH = os.path.dirname(__file__) + "/screenshot/"
#PATH = "C:\\work\\rex\\trunk\\bin\\pymodules\\webserver" + "/screenshot/"
PATH = "C:/work/rex/trunk/bin/pymodules/webserver/screenshot/"

print "WEB PATH for images:", PATH

class WebServer(Server):
    """this is the component that autoload registers to the viewer"""
    def __init__(self):
        Server.__init__(self, PORT)
        self + WebController() + Static(docroot=PATH)

class WebController(Controller):
    def index(self):
        return "Hello World!"

    def render(self):
        timestr = datetime.datetime.today().isoformat()
        imgname = "image.png"
        r.takeScreenshot(PATH, imgname)

        return """

Realxtend Naali viewer

at %s

        """ % (timestr, imgname)

#~ if __name__ == '__main__':
    #~ while 1:
        #~ print ".",
<|endoftext|>


<commit_before>"""a non-blocking, non-threaded non-multiprocessing circuits web server"""

import time #timestamping images
import datetime #showing human readable time on render page
import os

import rexviewer as r

try:
    import circuits
except ImportError: #not running within the viewer, but testing outside it
    import sys
    sys.path.append('..')

from circuits.web import Controller, Server, Static

#for camera rotating
import PythonQt.QtGui
from PythonQt.QtGui import QQuaternion as Quat
from PythonQt.QtGui import QVector3D as Vec

from objectedit.conversions import quat_to_euler #, euler_to_quat

PORT = 28008

#__file__ doesn't work in embedded context, but os.getcwd() helps
OWNPATH = os.getcwd() + "/pymodules/webserver/"
SHOTPATH = OWNPATH + "screenshot/"
#print "WEB PATH for images:", PATH

class WebServer(Server):
    """this is the component that autoload registers to the viewer"""
    def __init__(self):
        Server.__init__(self, "0.0.0.0:%d" % PORT) #"localhost", PORT)
        self + WebController() + Static(docroot=SHOTPATH)

#the first version with relative controls to the cam
relhtml = """\

  <title>Naali web ui</title>

Naali

  <form action="camcontrol" method="GET">

rotate:

  <input type="submit" name="rotate" value="10"/>
  <input type="submit" name="rotate" value="-10"/>

move:<br/
  <input type="submit" name="move" value="+1"/>

  <input type="submit" name="move" value="-1"/>

  </form>

"""

#second version where webui gives absolute pos&ort for the cam, so each user has own on client side
abshtml = open(OWNPATH + "webui.html").read()

def save_screenshot():
    imgname = "image-%s.png" % time.time()
    r.takeScreenshot(SHOTPATH, imgname)
    imgurl = imgname #base is added when the server has a separate http for serving the image
    return imgurl

class WebController(Controller):
    def index(self):
        return "Hello World!"

    def camcontrol(self, rotate=None, move=None):
        camid = r.getCameraId()
        #print "CAM:", camid
        cament = r.getEntity(camid)
        p = cament.placeable
        #print p.Position, p.Orientation

        if rotate is not None:
            ort = p.Orientation
            rot = Quat.fromAxisAndAngle(Vec(0, 1, 0), float(rotate))
            ort *= rot
            p.Orientation = ort

        if move is not None:
            pos = p.Position
            pos += Vec(float(move), 0, 0)
            p.Position = pos

        imgurl = save_screenshot()

        #return "%s, %s" % (p.Position, p.Orientation)
        return relhtml % imgurl

    def render(self, camposx=None, camposy=None, camposz=None, camang=None):
#, camortx=None, camorty=None, camortz=None, camortw=None):
        camid = r.getCameraId()
        cament = r.getEntity(camid)
        p = cament.placeable

        if camposx is not None:
            pos = Vec(*(float(v) for v in [camposx, camposy, camposz]))
            p.Position = pos

        if camang is not None:
            ort = p.Orientation
            start = Quat(0, 0, -0.707, -0.707)
            rot = Quat.fromAxisAndAngle(Vec(0, 1, 0), -float(camang))
            new = start * rot
            p.Orientation = new

        #if camortx is not None:
        #    ort = Quat(*(float(v) for v in [camortw, camortx, camorty, camortz]))
        #    p.Orientation = ort

        #return str(p.Position), str(p.Orientation) #self.render1()
        imgurl = save_screenshot()

        pos = p.Position
        ort = p.Orientation
        #vec, ang = toAngleAxis(p.Orientation)
        #print vec, ang
        euler = quat_to_euler([ort.scalar(), ort.x(), ort.y(), ort.z()])
        ang = euler[0]
        if ang < 0:
            ang = 360 + ang

        return abshtml % (imgurl,
                          ang,
                          pos.x(), pos.y(), pos.z()
                          #ort.scalar(), ort.x(), ort.y(), ort.z(),
                          )

    def render1(self, campos=None, camort=None):
        timestr = datetime.datetime.today().isoformat()
        imgurl = save_screenshot()

        return """

Realxtend Naali viewer

at %s

        """ % (timestr, imgurl)

#~ if __name__ == '__main__':
    #~ while 1:
        #~ print ".",
<commit_msg>a url handler for programmatic use: takes cam pos & ort vals in the http get, and returns the resulting img<commit_after>"""a non-blocking, non-threaded non-multiprocessing circuits web server"""

import time #timestamping images
import datetime #showing human readable time on render page
import os

import rexviewer as r

try:
    import circuits
except ImportError: #not running within the viewer, but testing outside it
    import sys
    sys.path.append('..')

from circuits.web import Controller, Server, Static

#for camera rotating
import PythonQt.QtGui
from PythonQt.QtGui import QQuaternion as Quat
from PythonQt.QtGui import QVector3D as Vec

from objectedit.conversions import quat_to_euler #, euler_to_quat

PORT = 28008

#__file__ doesn't work in embedded context, but os.getcwd() helps
OWNPATH = os.getcwd() + "/pymodules/webserver/"
SHOTPATH = OWNPATH + "screenshot/"
#print "WEB PATH for images:", PATH

class WebServer(Server):
    """this is the component that autoload registers to the viewer"""
    def __init__(self):
        Server.__init__(self, "0.0.0.0:%d" % PORT) #"localhost", PORT)
        self + WebController() + Static(docroot=SHOTPATH)

#the first version with relative controls to the cam
relhtml = """\

  <title>Naali web ui</title>

Naali

  <form action="camcontrol" method="GET">

rotate:

  <input type="submit" name="rotate" value="10"/>
  <input type="submit" name="rotate" value="-10"/>

move:<br/
  <input type="submit" name="move" value="+1"/>

  <input type="submit" name="move" value="-1"/>

  </form>

"""

#second version where webui gives absolute pos&ort for the cam, so each user has own on client side
abshtml = open(OWNPATH + "webui.html").read()

def save_screenshot():
    imgname = "image-%s.png" % time.time()
    r.takeScreenshot(SHOTPATH, imgname)
    imgurl = imgname #base is added when the server has a separate http for serving the image
    return imgurl

class WebController(Controller):
    def index(self):
        return "Hello World!"

    def camcontrol(self, rotate=None, move=None):
        camid = r.getCameraId()
        #print "CAM:", camid
        cament = r.getEntity(camid)
        p = cament.placeable
        #print p.Position, p.Orientation

        if rotate is not None:
            ort = p.Orientation
            rot = Quat.fromAxisAndAngle(Vec(0, 1, 0), float(rotate))
            ort *= rot
            p.Orientation = ort

        if move is not None:
            pos = p.Position
            pos += Vec(float(move), 0, 0)
            p.Position = pos

        imgurl = save_screenshot()

        #return "%s, %s" % (p.Position, p.Orientation)
        return relhtml % imgurl

    def render(self, camposx=None, camposy=None, camposz=None, camang=None):
#, camortx=None, camorty=None, camortz=None, camortw=None):
        camid = r.getCameraId()
        cament = r.getEntity(camid)
        p = cament.placeable

        if camposx is not None:
            pos = Vec(*(float(v) for v in [camposx, camposy, camposz]))
            p.Position = pos

        if camang is not None:
            ort = p.Orientation
            start = Quat(0, 0, -0.707, -0.707)
            rot = Quat.fromAxisAndAngle(Vec(0, 1, 0), -float(camang))
            new = start * rot
            p.Orientation = new

        #if camortx is not None:
        #    ort = Quat(*(float(v) for v in [camortw, camortx, camorty, camortz]))
        #    p.Orientation = ort

        #return str(p.Position), str(p.Orientation) #self.render1()
        imgurl = save_screenshot()

        pos = p.Position
        ort = p.Orientation
        #vec, ang = toAngleAxis(p.Orientation)
        #print vec, ang
        euler = quat_to_euler([ort.scalar(), ort.x(), ort.y(), ort.z()])
        ang = euler[0]
        if ang < 0:
            ang = 360 + ang

        return abshtml % (imgurl,
                          ang,
                          pos.x(), pos.y(), pos.z()
                          #ort.scalar(), ort.x(), ort.y(), ort.z(),
                          )

    def renderimg(self, camposx=None, camposy=None, camposz=None, camortx=None, camorty=None, camortz=None, camortw=None):
        camid = r.getCameraId()
        cament = r.getEntity(camid)
        p = cament.placeable

        if camposx is not None:
            pos = Vec(*(float(v) for v in [camposx, camposy, camposz]))
            p.Position = pos

        if camortx is not None:
            ort = Quat(*(float(v) for v in [camortw, camortx, camorty, camortz]))
            p.Orientation = ort

        imgurl = save_screenshot()
        #return imgurl
        return self.serve_file(SHOTPATH + imgurl)

    def render1(self, campos=None, camort=None):
        timestr = datetime.datetime.today().isoformat()
        imgurl = save_screenshot()

        return """

Realxtend Naali viewer

at %s

        """ % (timestr, imgurl)

#~ if __name__ == '__main__':
    #~ while 1:
        #~ print ".",
<|endoftext|>