<commit_before>"""
"""

import logging
import os
import mesos.interface
import progressbar
import StringIO
import tarfile
import tempfile
import threading
import uuid

from fnmatch import fnmatch
from functools import partial
from fs.opener import opener
from pesos.vendor.mesos import mesos_pb2
from urlparse import urlparse

from portainer.proto import portainer_pb2
from portainer.util.parser import parse_dockerfile, parse_dockerignore

logger = logging.getLogger("portainer.scheduler")

class Scheduler(mesos.interface.Scheduler):
    """Mesos scheduler that is responsible for launching the builder tasks."""

    def __init__(self, tasks, executor_uri, cpu_limit, mem_limit, push_registry,
                 staging_uri, stream=False, verbose=False, repository=None,
                 pull_registry=None, docker_host=None):

        self.executor_uri = executor_uri
        self.cpu = float(cpu_limit)
        self.mem = int(mem_limit)
        self.push_registry = push_registry
        self.pull_registry = pull_registry
        self.staging_uri = staging_uri
        self.stream = stream
        self.verbose = verbose
        self.repository = repository
        self.docker_host = docker_host

        self.queued_tasks = []
        for path, tags in tasks:
            dockerfile = parse_dockerfile(path, registry=pull_registry)
            self.queued_tasks.append((path, dockerfile, tags))

        self.pending = len(self.queued_tasks)
        self.running = 0
        self.finished = 0
        self.failed = 0
        self.task_ids = {}

        self.processing_offers = threading.Lock()

        # Ensure the staging directory exists
        self.filesystem = None
        if self.staging_uri:
            staging_uri = urlparse(self.staging_uri)
            staging_fs = opener.opendir(
                "%s://%s/" % (staging_uri.scheme, staging_uri.netloc)
            )

            staging_fs.makedir(
                staging_uri.path.lstrip("/"),
                recursive=True,
                allow_recreate=True
            )

            self.filesystem = opener.opendir(self.staging_uri)

    def registered(self, driver, frameworkId, masterInfo):
        host = masterInfo.hostname or masterInfo.ip
        master = "http://%s:%s" % (host, masterInfo.port)
        logger.info("Framework %s registered to %s", frameworkId.value, master)

    def disconnected(self, driver):
        logger.warning("Framework disconnected from the mesos master")

    def reregistered(self, driver, masterInfo):
        host = masterInfo.hostname or masterInfo.ip
        master = "http://%s:%s" % (host, masterInfo.port)
        logger.info("Framework re-registered to %s", master)

    def error(self, driver, message):
        logger.error("Framework error: %s", message)

    def resource_offers(self, driver, offers):

        # Spawn another thread to handle offer processing to free up the driver
        t = threading.Thread(target=partial(
            self._handle_offers,
            driver,
            offers
        ))

        t.setDaemon(True)
        t.start()

    def _handle_offers(self, driver, offers):

        # We only want to process offers one set at a time
        with self.processing_offers:
            decline_offers = []
            launch_offers = []

            if not self.pending:
                decline_offers = [offer.id for offer in offers]
            else:
                for offer in offers:
                    offer_cpu = 0.0
                    offer_mem = 0

                    for resource in offer.resources:
                        if resource.name == "cpus":
                            offer_cpu = float(resource.scalar.value)
                        if resource.name == "mem":
                            offer_mem = int(resource.scalar.value)

                    logger.debug("Received offer for cpus:%f mem:%d", offer_cpu, offer_mem)

                    # Look for a task in the queue that fits the bill
                    decline_offer = True
                    for idx, (path, dockerfile, tags) in enumerate(self.queued_tasks):
                        cpu = float(dockerfile.get("BUILD_CPU", [self.cpu]).next()[0])
                        mem = int(dockerfile.get("BUILD_MEM", [self.mem]).next()[0])

                        if cpu > offer_cpu:
                            break
                        if mem <= offer_mem:
                            decline_offer = False
                            launch_offers.append((offer, idx, cpu, mem))
                            break  # TODO: No support for multiple tasks per offer yet

                    if decline_offer:
                        logger.debug("Ignoring offer %r", offer)
                        decline_offers.append(offer.id)

            if decline_offers:
                driver.declineOffer(decline_offers)

            if launch_offers:
                tasks = []
                task_queue = list(self.queued_tasks)

                for offer, idx, cpu, mem in launch_offers:
                    path, dockerfile, tags = task_queue[idx]
                    task_queue[idx] = None  # Null out the value to be removed later

                    tasks.append(self._prepare_task(
                        driver=driver,
                        path=path,
                        dockerfile=dockerfile,
                        tags=tags,
                        offer=offer,
                        cpu=cpu,
                        mem=mem
                    ))

                self.queued_tasks = filter(None, task_queue)
                self.pending -= len(tasks)
                self.running += len(tasks)

                logger.info("Launching %d tasks", len(tasks))
                driver.launchTasks(offer.id, tasks)

    def status_update(self, driver, update):
        """Called when a status update is received from the mesos cluster."""

        finished = False
        failed = False
        task_id = None

        if update.task_id.value in self.task_ids:
            build_task = self.task_ids[update.task_id.value]
            task_id = build_task.image.repository
        else:
            task_id = update.task_id.value
            logger.error("Task update for unknown task! %s", task_id)

        if update.state == mesos_pb2.TASK_STARTING:
            logger.info("Task update %s : STARTING", task_id)
        if update.state == mesos_pb2.TASK_RUNNING:
            logger.info("Task update %s : RUNNING", task_id)
        if update.state == mesos_pb2.TASK_FAILED:
            logger.info("Task update %s : FAILED", task_id)
            if update.message and update.data:
                logger.info("Exception caught while building image: \n\n%s", update.data)
            failed = True
        elif update.state == mesos_pb2.TASK_FINISHED:
            logger.info("Task update %s : FINISHED", task_id)
            finished = True
        elif update.state == mesos_pb2.TASK_KILLED:
            logger.info("Task update %s : KILLED", task_id)
            failed = True
        elif update.state == mesos_pb2.TASK_LOST:
            logger.info("Task update %s : LOST", task_id)
            failed = True

        if finished:
            self.running -= 1
            self.finished += 1
        elif failed:
            self.running -= 1
            self.failed += 1

        # If there are no tasks running, and the queue is empty, we should stop
        if self.running == 0 and self.pending == 0:
            driver.stop()

    def framework_message(self, driver, executorId, slaveId, message):
        if "Buffering" in message:  # Heh. This'll do for now, eh?
            logger.debug("\t%s", message)
        else:
            logger.info("\t%s", message)

    def _prepare_task(self, driver, path, dockerfile, tags, offer, cpu, mem):
        """Prepare a given dockerfile build task atop the given mesos offer."""

        # Generate a task ID
        task_id = str(uuid.uuid1())
        logger.info("Preparing task %s to build %s", task_id, path)

        # Define the build that's required
        build_task = portainer_pb2.BuildTask()
        build_task.stream = self.stream

        # Create a custom docker context if there are local sources
        staging_context_path = None
        if dockerfile.has_local_sources:
            working_dir = os.path.abspath(os.path.dirname(path))

            # Generate the dockerfile build context
            _, context_path = tempfile.mkstemp()
            context = open(context_path, "w+b")

            logger.debug("Writing context tar to %s", context_path)
            context_size = self._make_build_context(context, working_dir, dockerfile)

            # Put together the staging directory
            staging_dir = os.path.join("staging", task_id)
            context_filename = "docker_context.tar.gz"

            staging_context_path = os.path.join(staging_dir, context_filename)

            # Create the directory
            logger.debug("Task staging directory %s", staging_dir)
            self.filesystem.makedir(staging_dir, recursive=True)

            # Upload the build context (+ fancy progress bar)
            logger.info("Uploading context (%d bytes)", context_size)
            pbar = progressbar.ProgressBar(maxval=context_size, term_width=100)

            # Define a custom error handler for the async upload
            def handle_exception(e):
                logger.error("Caught exception uploading the context")
                raise e

            event = self.filesystem.setcontents_async(
                path=staging_context_path,
                data=context,
                progress_callback=pbar.update,
                finished_callback=pbar.finish,
                error_callback=handle_exception
            )

            # Hold up, let's wait until the upload finishes
            event.wait()

            # Close and clear up the tmp context
            logger.debug("Cleaning up local context %s", context_path)
            context.close()
            os.unlink(context_path)

            build_task.context = context_filename
        else:
            build_task.dockerfile = dockerfile.build()

        if self.docker_host:
            build_task.docker_host = self.docker_host

        # Pull out the repository from the dockerfile
        try:
            build_task.image.repository = dockerfile.get("REPOSITORY", [self.repository]).next()[0]
        except (StopIteration, IndexError):
            raise ValueError("No REPOSITORY given for %s", path)

        # Pull out the registry from the dockerfile
        try:
            registry = self.push_registry.split(":")
            build_task.image.registry.hostname = registry[0]
            if len(registry) > 1:
                build_task.image.registry.port = int(registry[1])
        except ValueError:
            raise ValueError("Failed to parse REGISTRY in %s", path)

        # Add any tags
        build_task.image.tag.extend(tags)

        # Define the mesos task
        task = mesos_pb2.TaskInfo()
        task.name = "%s/%s" % (":".join(registry), build_task.image.repository)
        task.task_id.value = task_id
        task.slave_id.value = offer.slave_id.value

        # Create the executor
        args = []
        if self.verbose:
            args.append("--verbose")

        task.executor.executor_id.value = task_id
        task.executor.command.value = "./%s/bin/portainer %s build-executor" % (
            os.path.basename(self.executor_uri).rstrip(".tar.gz"), " ".join(args)
        )

        # TODO(tarnfeld): Make this configurable
        # TODO(tarnfeld): Support the mesos 0.20.0 docker protobuf
        task.executor.command.container.image = "docker://jpetazzo/dind"

        # We have to mount the /var/lib/docker VOLUME inside of the sandbox
        task.executor.command.container.options.extend(["--privileged"])
        task.executor.command.container.options.extend(["-v", "$MESOS_DIRECTORY/docker:/var/lib/docker"])

        task.executor.name = "build"
        task.executor.source = "portainer"

        # Configure the mesos executor with the portainer executor uri
        portainer_executor = task.executor.command.uris.add()
        portainer_executor.value = self.executor_uri

        if staging_context_path:
            # Add the docker context
            uri = task.executor.command.uris.add()
            uri.value = os.path.join(self.staging_uri, staging_context_path)
            uri.extract = False

        task.data = build_task.SerializeToString()
        task.executor.data = task.data

        # Build up the resources
        cpu_resource = task.resources.add()
        cpu_resource.name = "cpus"
        cpu_resource.type = mesos_pb2.Value.SCALAR
        cpu_resource.scalar.value = cpu

        mem_resource = task.resources.add()
        mem_resource.name = "mem"
        mem_resource.type = mesos_pb2.Value.SCALAR
        mem_resource.scalar.value = mem

        self.task_ids[task_id] = build_task

        logger.info("Prepared task %s to build %s", task_id, path)
        logger.debug("%s", build_task)

        return task

    def _make_build_context(self, output, context_root, dockerfile):
        """Generate and return a compressed tar archive of the build context."""

        if not self.filesystem:
            raise Exception("A staging filesystem is required for local sources")

        tar = tarfile.open(mode="w:gz", fileobj=output)
        for idx, (cmd, instruction) in enumerate(dockerfile.instructions):
            if cmd == "ADD":
                local_path, remote_path = instruction
                tar_path = "context/%s" % str(idx)

                # TODO(tarnfeld): This isn't strict enough
                if local_path.startswith("http"):
                    logger.debug("Skipping remote ADD %s", local_path)
                    continue

                if not local_path.startswith("/"):
                    local_path = os.path.join(context_root, local_path)
                local_path = os.path.abspath(local_path)

                if os.path.isfile(local_path):
                    # Preserve the file extension
                    parts = local_path.split(".")
                    if len(parts) > 1:
                        tar_path += "." + parts[-1]
                    logger.debug("Adding path %s to tar in %s", local_path, tar_path)
                    tar.add(local_path, arcname=tar_path)
                else:
                    ignore = set()
                    for (dirpath, _, filenames) in os.walk(local_path, followlinks=True):
                        # Update the set of ignored paths with any new .dockerignore files we see
                        ignore_path = os.path.join(dirpath, ".dockerignore")
                        if os.path.exists(ignore_path):
                            with open(ignore_path, 'r') as f:
                                for glob in parse_dockerignore(f):
                                    ignore.add(os.path.join(dirpath, glob) + "*")

                        for filename in filenames:
                            path = os.path.join(dirpath, filename)
                            for expr in ignore:
                                if fnmatch(path, expr):
                                    logger.debug("Ignoring path %s", path)
                                    break
                            else:
                                rel_path = path.replace(local_path, '').lstrip('/')
                                logger.debug("Adding path %s to tar in %s", rel_path, tar_path)
                                tar.add(path, arcname=os.path.join(tar_path, rel_path))

                dockerfile.instructions[idx] = (cmd, (tar_path, remote_path))

        # Write the modified dockerfile into the tar also
        buildfile = StringIO.StringIO()
        buildfile.write("# Generated by portainer\n")

        for cmd, instructions in dockerfile.instructions:
            if cmd not in dockerfile.INTERNAL:
                line = "%s %s" % (cmd, " ".join(instructions))

                logger.debug("Adding instruction %r to dockerfile", line)
                buildfile.write("%s\n" % line)

        buildfile.seek(0, os.SEEK_END)
        info = tarfile.TarInfo("Dockerfile")
        info.size = buildfile.tell()

        buildfile.seek(0)
        tar.addfile(info, fileobj=buildfile)

        tar.close()
        output.seek(0, os.SEEK_END)
        tar_size = output.tell()
        output.seek(0)

        return tar_size
<commit_msg>Refactor task launching to be a bit cleaner<commit_after>"""
"""

import logging
import os
import mesos.interface
import progressbar
import StringIO
import tarfile
import tempfile
import threading
import uuid

from fnmatch import fnmatch
from functools import partial
from fs.opener import opener
from pesos.vendor.mesos import mesos_pb2
from urlparse import urlparse

from portainer.proto import portainer_pb2
from portainer.util.parser import parse_dockerfile, parse_dockerignore

logger = logging.getLogger("portainer.scheduler")

class Scheduler(mesos.interface.Scheduler):
    """Mesos scheduler that is responsible for launching the builder tasks."""

    def __init__(self, tasks, executor_uri, cpu_limit, mem_limit, push_registry,
                 staging_uri, stream=False, verbose=False, repository=None,
                 pull_registry=None, docker_host=None):

        self.executor_uri = executor_uri
        self.cpu = float(cpu_limit)
        self.mem = int(mem_limit)
        self.push_registry = push_registry
        self.pull_registry = pull_registry
        self.staging_uri = staging_uri
        self.stream = stream
        self.verbose = verbose
        self.repository = repository
        self.docker_host = docker_host

        self.queued_tasks = []
        for path, tags in tasks:
            dockerfile = parse_dockerfile(path, registry=pull_registry)
            self.queued_tasks.append((path, dockerfile, tags))

        self.pending = len(self.queued_tasks)
        self.running = 0
        self.finished = 0
        self.failed = 0
        self.task_ids = {}

        self.processing_offers = threading.Lock()

        # Ensure the staging directory exists
        self.filesystem = None
        if self.staging_uri:
            staging_uri = urlparse(self.staging_uri)
            staging_fs = opener.opendir(
                "%s://%s/" % (staging_uri.scheme, staging_uri.netloc)
            )

            staging_fs.makedir(
                staging_uri.path.lstrip("/"),
                recursive=True,
                allow_recreate=True
            )

            self.filesystem = opener.opendir(self.staging_uri)

    def registered(self, driver, frameworkId, masterInfo):
        host = masterInfo.hostname or masterInfo.ip
        master = "http://%s:%s" % (host, masterInfo.port)
        logger.info("Framework %s registered to %s", frameworkId.value, master)

    def disconnected(self, driver):
        logger.warning("Framework disconnected from the mesos master")

    def reregistered(self, driver, masterInfo):
        host = masterInfo.hostname or masterInfo.ip
        master = "http://%s:%s" % (host, masterInfo.port)
        logger.info("Framework re-registered to %s", master)

    def error(self, driver, message):
        logger.error("Framework error: %s", message)

    def resource_offers(self, driver, offers):

        # Spawn another thread to handle offer processing to free up the driver
        t = threading.Thread(target=partial(
            self._handle_offers,
            driver,
            offers
        ))

        t.setDaemon(True)
        t.start()

    def _handle_offers(self, driver, offers):

        # We only want to process offers one set at a time
        with self.processing_offers:
            decline_offers = []
            tasks_to_launch = []

            if not self.pending:
                decline_offers = [offer.id for offer in offers]
            else:
                for offer in offers:
                    offer_cpu = 0.0
                    offer_mem = 0

                    # Extract the important resources from the offer
                    for resource in offer.resources:
                        if resource.name == "cpus":
                            offer_cpu = float(resource.scalar.value)
                        if resource.name == "mem":
                            offer_mem = int(resource.scalar.value)

                    logger.debug("Received offer for cpus:%f mem:%d", offer_cpu, offer_mem)

                    # Look for a task in the queue that fits the bill
                    for idx, (path, dockerfile, tags) in enumerate(self.queued_tasks):
                        cpu = float(dockerfile.get("BUILD_CPU", [self.cpu]).next()[0])
                        mem = int(dockerfile.get("BUILD_MEM", [self.mem]).next()[0])

                        if cpu <= offer_cpu and mem <= offer_mem:
                            self.queued_tasks[idx] = None  # Remove the task from the queue
                            self.pending -= 1
                            self.running += 1
                            tasks_to_launch.append((offer, path, dockerfile,
                                                    tags, cpu, mem))
                            # TODO: No support for multiple tasks per offer yet
                            break
                    else:
                        logger.debug("Ignoring offer %r", offer)
                        decline_offers.append(offer.id)

                    # Remove all of the tasks that are about to be launched
                    self.queued_tasks = filter(None, self.queued_tasks)

            if decline_offers:
                driver.declineOffer(decline_offers)

            # Launch the build tasks on the mesos cluster
            for offer, path, dockerfile, tags, cpu, mem in tasks_to_launch:
                tasks = [self._prepare_task(
                    driver=driver,
                    path=path,
                    dockerfile=dockerfile,
                    tags=tags,
                    offer=offer,
                    cpu=cpu,
                    mem=mem
                )]

                logger.info("Launching %d tasks", len(tasks))
                driver.launchTasks(offer.id, tasks)

    def status_update(self, driver, update):
        """Called when a status update is received from the mesos cluster."""

        finished = False
        failed = False
        task_id = None

        if update.task_id.value in self.task_ids:
            build_task = self.task_ids[update.task_id.value]
            task_id = build_task.image.repository
        else:
            task_id = update.task_id.value
            logger.error("Task update for unknown task! %s", task_id)

        if update.state == mesos_pb2.TASK_STARTING:
            logger.info("Task update %s : STARTING", task_id)
        if update.state == mesos_pb2.TASK_RUNNING:
            logger.info("Task update %s : RUNNING", task_id)
        if update.state == mesos_pb2.TASK_FAILED:
            logger.info("Task update %s : FAILED", task_id)
            if update.message and update.data:
                logger.info("Exception caught while building image: \n\n%s", update.data)
            failed = True
        elif update.state == mesos_pb2.TASK_FINISHED:
            logger.info("Task update %s : FINISHED", task_id)
            finished = True
        elif update.state == mesos_pb2.TASK_KILLED:
            logger.info("Task update %s : KILLED", task_id)
            failed = True
        elif update.state == mesos_pb2.TASK_LOST:
            logger.info("Task update %s : LOST", task_id)
            failed = True

        if finished:
            self.running -= 1
            self.finished += 1
        elif failed:
            self.running -= 1
            self.failed += 1

        # If there are no tasks running, and the queue is empty, we should stop
        if self.running == 0 and self.pending == 0:
            driver.stop()

    def framework_message(self, driver, executorId, slaveId, message):
        if "Buffering" in message:  # Heh. This'll do for now, eh?
            logger.debug("\t%s", message)
        else:
            logger.info("\t%s", message)

    def _prepare_task(self, driver, path, dockerfile, tags, offer, cpu, mem):
        """Prepare a given dockerfile build task atop the given mesos offer."""

        # Generate a task ID
        task_id = str(uuid.uuid1())
        logger.info("Preparing task %s to build %s", task_id, path)

        # Define the build that's required
        build_task = portainer_pb2.BuildTask()
        build_task.stream = self.stream

        # Create a custom docker context if there are local sources
        staging_context_path = None
        if dockerfile.has_local_sources:
            working_dir = os.path.abspath(os.path.dirname(path))

            # Generate the dockerfile build context
            _, context_path = tempfile.mkstemp()
            context = open(context_path, "w+b")

            logger.debug("Writing context tar to %s", context_path)
            context_size = self._make_build_context(context, working_dir, dockerfile)

            # Put together the staging directory
            staging_dir = os.path.join("staging", task_id)
            context_filename = "docker_context.tar.gz"

            staging_context_path = os.path.join(staging_dir, context_filename)

            # Create the directory
            logger.debug("Task staging directory %s", staging_dir)
            self.filesystem.makedir(staging_dir, recursive=True)

            # Upload the build context (+ fancy progress bar)
            logger.info("Uploading context (%d bytes)", context_size)
            pbar = progressbar.ProgressBar(maxval=context_size, term_width=100)

            # Define a custom error handler for the async upload
            def handle_exception(e):
                logger.error("Caught exception uploading the context")
                raise e

            event = self.filesystem.setcontents_async(
                path=staging_context_path,
                data=context,
                progress_callback=pbar.update,
                finished_callback=pbar.finish,
                error_callback=handle_exception
            )

            # Hold up, let's wait until the upload finishes
            event.wait()

            # Close and clear up the tmp context
            logger.debug("Cleaning up local context %s", context_path)
            context.close()
            os.unlink(context_path)

            build_task.context = context_filename
        else:
            build_task.dockerfile = dockerfile.build()

        if self.docker_host:
            build_task.docker_host = self.docker_host

        # Pull out the repository from the dockerfile
        try:
            build_task.image.repository = dockerfile.get("REPOSITORY", [self.repository]).next()[0]
        except (StopIteration, IndexError):
            raise ValueError("No REPOSITORY given for %s", path)

        # Pull out the registry from the dockerfile
        try:
            registry = self.push_registry.split(":")
            build_task.image.registry.hostname = registry[0]
            if len(registry) > 1:
                build_task.image.registry.port = int(registry[1])
        except ValueError:
            raise ValueError("Failed to parse REGISTRY in %s", path)

        # Add any tags
        build_task.image.tag.extend(tags)

        # Define the mesos task
        task = mesos_pb2.TaskInfo()
        task.name = "%s/%s" % (":".join(registry), build_task.image.repository)
        task.task_id.value = task_id
        task.slave_id.value = offer.slave_id.value

        # Create the executor
        args = []
        if self.verbose:
            args.append("--verbose")

        task.executor.executor_id.value = task_id
        task.executor.command.value = "./%s/bin/portainer %s build-executor" % (
            os.path.basename(self.executor_uri).rstrip(".tar.gz"), " ".join(args)
        )

        # TODO(tarnfeld): Make this configurable
        # TODO(tarnfeld): Support the mesos 0.20.0 docker protobuf
        task.executor.command.container.image = "docker://jpetazzo/dind"

        # We have to mount the /var/lib/docker VOLUME inside of the sandbox
        task.executor.command.container.options.extend(["--privileged"])
        task.executor.command.container.options.extend(["-v", "$MESOS_DIRECTORY/docker:/var/lib/docker"])

        task.executor.name = "build"
        task.executor.source = "portainer"

        # Configure the mesos executor with the portainer executor uri
        portainer_executor = task.executor.command.uris.add()
        portainer_executor.value = self.executor_uri

        if staging_context_path:
            # Add the docker context
            uri = task.executor.command.uris.add()
            uri.value = os.path.join(self.staging_uri, staging_context_path)
            uri.extract = False

        task.data = build_task.SerializeToString()
        task.executor.data = task.data

        # Build up the resources
        cpu_resource = task.resources.add()
        cpu_resource.name = "cpus"
        cpu_resource.type = mesos_pb2.Value.SCALAR
        cpu_resource.scalar.value = cpu

        mem_resource = task.resources.add()
        mem_resource.name = "mem"
        mem_resource.type = mesos_pb2.Value.SCALAR
        mem_resource.scalar.value = mem

        self.task_ids[task_id] = build_task

        logger.info("Prepared task %s to build %s", task_id, path)
        logger.debug("%s", build_task)

        return task

    def _make_build_context(self, output, context_root, dockerfile):
        """Generate and return a compressed tar archive of the build context."""

        if not self.filesystem:
            raise Exception("A staging filesystem is required for local sources")

        tar = tarfile.open(mode="w:gz", fileobj=output)
        for idx, (cmd, instruction) in enumerate(dockerfile.instructions):
            if cmd == "ADD":
                local_path, remote_path = instruction
                tar_path = "context/%s" % str(idx)

                # TODO(tarnfeld): This isn't strict enough
                if local_path.startswith("http"):
                    logger.debug("Skipping remote ADD %s", local_path)
                    continue

                if not local_path.startswith("/"):
                    local_path = os.path.join(context_root, local_path)
                local_path = os.path.abspath(local_path)

                if os.path.isfile(local_path):
                    # Preserve the file extension
                    parts = local_path.split(".")
                    if len(parts) > 1:
                        tar_path += "." + parts[-1]
                    logger.debug("Adding path %s to tar in %s", local_path, tar_path)
                    tar.add(local_path, arcname=tar_path)
                else:
                    ignore = set()
                    for (dirpath, _, filenames) in os.walk(local_path, followlinks=True):
                        # Update the set of ignored paths with any new .dockerignore files we see
                        ignore_path = os.path.join(dirpath, ".dockerignore")
                        if os.path.exists(ignore_path):
                            with open(ignore_path, 'r') as f:
                                for glob in parse_dockerignore(f):
                                    ignore.add(os.path.join(dirpath, glob) + "*")

                        for filename in filenames:
                            path = os.path.join(dirpath, filename)
                            for expr in ignore:
                                if fnmatch(path, expr):
                                    logger.debug("Ignoring path %s", path)
                                    break
                            else:
                                rel_path = path.replace(local_path, '').lstrip('/')
                                logger.debug("Adding path %s to tar in %s", rel_path, tar_path)
                                tar.add(path, arcname=os.path.join(tar_path, rel_path))

                dockerfile.instructions[idx] = (cmd, (tar_path, remote_path))

        # Write the modified dockerfile into the tar also
        buildfile = StringIO.StringIO()
        buildfile.write("# Generated by portainer\n")

        for cmd, instructions in dockerfile.instructions:
            if cmd not in dockerfile.INTERNAL:
                line = "%s %s" % (cmd, " ".join(instructions))

                logger.debug("Adding instruction %r to dockerfile", line)
                buildfile.write("%s\n" % line)

        buildfile.seek(0, os.SEEK_END)
        info = tarfile.TarInfo("Dockerfile")
        info.size = buildfile.tell()

        buildfile.seek(0)
        tar.addfile(info, fileobj=buildfile)

        tar.close()
        output.seek(0, os.SEEK_END)
        tar_size = output.tell()
        output.seek(0)

        return tar_size
<|endoftext|>


<commit_before>#!/usr/bin/python

import numpy as np
from Locus import Locus

class Chrom(object):
    def __init__(self,id,length):
        self.id = id
        self.length = length

    def rQTL(self,length):
        ''' returns a random QTL within the chromosome '''
        start = np.random.randint(0,self.length)
        while start+length > self.length:
            start = np.random.randint(0,self.length)
        return Locus(chrom=self.id,start=start,end=start+length,id="RQTL-{}".format(length))

    def rLocus(self):
        ''' returns a random Locus from within the chromosome '''
        pos = np.random.randint(0,self.length)
        return Locus(chrom=self.id,start=pos,end=pos,id='rLocus-chr{}:{}'.format(self.id,pos))

    def __len__(self):
        return self.length

    def __str__(self):
        return "Chromosome {}: {} bp".format(self.id,self.length)

    def __repr__(self):
        return str(self)

<commit_msg>clafified language of random sampling from chromosome class<commit_after>#!/usr/bin/python

import numpy as np
from Locus import Locus

class Chrom(object):
    def __init__(self,id,length):
        self.id = id
        self.length = length

    def rLocus(self,length):
        ''' returns a random QTL within the chromosome '''
        start = np.random.randint(0,self.length)
        while start+length > self.length:
            start = np.random.randint(0,self.length)
        return Locus(chrom=self.id,start=start,end=start+length,id="rLocus-{}".format(length))

    def rSNP(self):
        ''' returns a random single nucleotide (position) from within the chromosome '''
        pos = np.random.randint(0,self.length)
        return Locus(chrom=self.id,start=pos,end=pos,id='rSNP-chr{}:{}'.format(self.id,pos))

    def __len__(self):
        return self.length

    def __str__(self):
        return "Chromosome {}: {} bp".format(self.id,self.length)

    def __repr__(self):
        return str(self)

<|endoftext|>


<commit_before>"""This script logs into a user's reddit account, accesses
     one of their subreddits, collects all user flair, and
     builds a browse-able log of users grouped by flair
"""
import praw
import Config
import Dictionary

def main():
	"""Log into reddit, get subreddit data, print log to stdout
	"""
	r = praw.Reddit(user_agent=Config.user_agent)
	r.login(Config.username, Config.password)
	subreddit = r.get_subreddit(Config.subreddit)
	collect = {}
	for flair in subreddit.get_flair_list(limit=None):
		(_, user, text) = flair.values()
		cleaned = text
		# flair pre-processing
		if cleaned.count(" ") >= 2:
			cleaned = " ".join(cleaned.split(" ")[0:2])
		postfix = ["'", "(", "-"]
		for char in postfix:
			if char in cleaned:
				cleaned = cleaned[:cleaned.index(char)]
		cleaned = cleaned.strip().lower()
		if cleaned in Dictionary.translate:
			college = Dictionary.translate[cleaned]
		elif cleaned == "":
			college = "{ Flair not set }"
		else:
			college = "{ Flair not recognized }"
			user = user + ' (' + (cleaned).encode('ascii', 'replace') + ')'
		if college in collect:
			collect[college].append('/u/' + user)
		else:
			collect[college] = ['/u/' + user]
	for college in sorted(collect):
		print "###", link(college), "\n*", "\n* ".join(collect[college]), "\n"

def link(college):
	"""string -> string
	    Sigma Chi customization: convert headers into FB links
	    Users can click on the name of a school to find friends
	"""
	if '{' in college:
		return college
	html = clean(college)
	fof = "https://www.facebook.com/search/people/?q=friends%20of%20my%20friends%20who%20are%20men%20and%20go%20to%20{}%20and%20like%20Sigma%20Chi%20Fraternity"
	return "[{}]({})".format(college, fof.format(html))

def clean(text):
	"""string -> string
	    remove non-compliant HTML text from school strings
	"""
	array = list(text)
	cleaned = []
	while len(array) > 0:
		char = array.pop(0)
		if char == " ":
			cleaned.append("%20")
		elif char == "(" or char == "[":
			counter = 1
			char = array.pop(0)
			while len(array) > 0 and counter >= 1:
				if char == "(" or char == "[":
					counter += 1
				elif char == ")" or char == "]":
					counter -= 1
				char = array.pop(0)
		else:
			cleaned.append(char)
	return "".join(cleaned)

if __name__ == "__main__":
	try:
		main()
	except SystemError:
		print "Bot was killed."

<commit_msg>break up Facebook query into multiple string literals<commit_after>"""This script logs into a user's reddit account, accesses
     one of their subreddits, collects all user flair, and
     builds a browse-able log of users grouped by flair
"""
import praw
import Config
import Dictionary

def main():
	"""Log into reddit, get subreddit data, print log to stdout
	"""
	r = praw.Reddit(user_agent=Config.user_agent)
	r.login(Config.username, Config.password)
	subreddit = r.get_subreddit(Config.subreddit)
	collect = {}
	for flair in subreddit.get_flair_list(limit=None):
		(_, user, text) = flair.values()
		cleaned = text
		# flair pre-processing
		if cleaned.count(" ") >= 2:
			cleaned = " ".join(cleaned.split(" ")[0:2])
		postfix = ["'", "(", "-"]
		for char in postfix:
			if char in cleaned:
				cleaned = cleaned[:cleaned.index(char)]
		cleaned = cleaned.strip().lower()
		if cleaned in Dictionary.translate:
			college = Dictionary.translate[cleaned]
		elif cleaned == "":
			college = "{ Flair not set }"
		else:
			college = "{ Flair not recognized }"
			user = user + ' (' + (cleaned).encode('ascii', 'replace') + ')'
		if college in collect:
			collect[college].append('/u/' + user)
		else:
			collect[college] = ['/u/' + user]
	for college in sorted(collect):
		print "###", link(college), "\n*", "\n* ".join(collect[college]), "\n"

def link(college):
	"""string -> string
	    Sigma Chi customization: convert headers into FB links
	    Users can click on the name of a school to find friends
	"""
	if '{' in college:
		return college
	html = clean(college)
	fof = ("https://www.facebook.com/search/people/"
	       "?q=friends%20of%20my%20friends%20"
	       "who%20are%20men%20and%20go%20to%20{}%20"
	       "and%20like%20Sigma%20Chi%20Fraternity")
	return "[{}]({})".format(college, fof.format(html))

def clean(text):
	"""string -> string
	    remove non-compliant HTML text from school strings
	"""
	array = list(text)
	cleaned = []
	while len(array) > 0:
		char = array.pop(0)
		if char == " ":
			cleaned.append("%20")
		elif char == "(" or char == "[":
			counter = 1
			char = array.pop(0)
			while len(array) > 0 and counter >= 1:
				if char == "(" or char == "[":
					counter += 1
				elif char == ")" or char == "]":
					counter -= 1
				char = array.pop(0)
		else:
			cleaned.append(char)
	return "".join(cleaned)

if __name__ == "__main__":
	try:
		main()
	except SystemError:
		print "Bot was killed."

<|endoftext|>


<commit_before>from collections import MutableMapping
from pymongo import MongoClient, Connection
from bson import ObjectId
from collections import Mapping
from cherrycommon.dictutils import MappingView, dump_value, Diffed
from cherrycommon.timeutils import seconds

DEFAULT_HOST = 'localhost'
DEFAULT_PORT = 27017
DEFAULT_DB_VERSION = 1

_mongo_clients = {}

def get_mongo_client(host=DEFAULT_HOST, port=DEFAULT_PORT, **kwargs):
    """Return (cached) MongoClient instance for provided URL.

    :param host: Host to connect.
    :type host: basestring
    :param port: Port to connect.
    :type port: int
    :rtype: MongoClient
    :return: Instance of MongoClient for provided host - port pair.
    """
    url = '{}:{}'.format(host, port)
    try:
        return _mongo_clients[(host, port)]
    except KeyError:
        client = _mongo_clients[(host, port)] = MongoClient(host, port, **kwargs)
        return client

class DataProvider(Mapping):
    def _get_collection(self, host, port, db, collection):
        try:
            return self._collection
        except AttributeError:
            self._collection = get_mongo_client(host, port)[db][collection]
            return self._collection

    @property
    def collection(self):
        return self._collection

    _global_cache = {}

    @classmethod
    def _get_cache(cls, db, collection):
        return cls._global_cache.setdefault((db, collection), {})

    _index_ttl = seconds(hours=1)

    def __init__(self, db, collection, use_cache=False, indexes=None, host=DEFAULT_HOST, port=DEFAULT_PORT):
        self._db_name = db
        self._collection_name = collection
        self._host = host
        self._port = port
        self._collection = self._get_collection(host, port, db, collection)

        if indexes is not None:
            for index_name in indexes:
                self._collection.ensure_index(index_name, ttl=self._index_ttl)

        if use_cache:
            self._cache = self._get_cache(db, collection)
        else:
            self._cache = None

    def _drop_cache_entry(self, pk):
        if self._cache:
            try:
                del self._cache[pk]
            except KeyError:
                pass

    def _prepare_fields(self, include_fields, exclude_fields):
        if self._cache:
            return None
        if include_fields:
            fields = dict.fromkeys(include_fields, 1)
            return fields
        if exclude_fields:
            return dict.fromkeys(exclude_fields, 0)

    @property
    def use_cache(self):
        return self._cache is not None

    def get(self, _id, include_fields=None, exclude_fields=None, force_reload=False):
        """
        Get document from collection by its primary key. 'fields' argument does not matter,
        if DataProvider caches it's data.
        """
        fields = self._prepare_fields(include_fields, exclude_fields)
        if self.use_cache and (not force_reload):
            try:
                return self._cache[_id]
            except KeyError:
                pass
        document = self._collection.find_one(_id, fields=fields)
        if self.use_cache:
            if document:
                self._cache[_id] = document
            else:
                self._drop_cache_entry(_id)
        return document

    @staticmethod
    def _keys_iterator(cursor):
        for document in cursor:
            yield document['_id'], document

    def find(self, *args, **kwargs):
        """Searches collection for documents, that matches filters. Cache is ignored for this operation.
        """
        include_fields = kwargs.pop('include_fields', {})
        exclude_fields = kwargs.pop('exclude_fields', {})
        kwargs['fields'] = kwargs.get('fields') or self._prepare_fields(include_fields, exclude_fields)
        keys = kwargs.pop('keys', False)
        cursor = self._collection.find(*args, **kwargs)
        if keys:
            return self._keys_iterator(cursor)
        else:
            return cursor

    def find_and_modify(self, *args, **kwargs):
        """Executes find and modify against the collection.
        """
        return self._collection.find_and_modify(*args, **kwargs)

    def find_one(self, spec, *args, **kwargs):
        include_fields = kwargs.pop('include_fields', {})
        exclude_fields = kwargs.pop('exclude_fields', {})
        kwargs['fields'] = kwargs.get('fields') or self._prepare_fields(include_fields, exclude_fields)
        if not isinstance(spec, dict):
            spec = {'_id': spec}
        return self._collection.find_one(spec, *args, **kwargs)

    def all(self, include_fields=None, exclude_fields=None, keys=False, *args, **kwargs):
        """Return cursor with all documents in collection.

        :rtype: Cursor
        """
        return self.find(include_fields=include_fields, exclude_fields=exclude_fields, keys=keys, *args, **kwargs)

    def ids(self):
        """List ids of all documents in collection
        """
        return self._collection.distinct('_id')

    def save(self, document, safe=False):
        """Saves document in collection. Creates one, if not exists yet.
        """
        self._collection.save(document, safe=safe)
        if hasattr(document, '_id'):
            self._drop_cache_entry(document['_id'])

    def insert(self, documents, **kwargs):
        """Stores documents into the collection.

        :param documents: document or list of documents to store.
        """
        if not isinstance(documents, list):
            documents = [documents]
        self._collection.insert(documents, **kwargs)

    def update(self, query, update, **kwargs):
        """Updates documents in collection.
        You can also pass named args, supported by pymongo.Collection.update method.
        Warning! Update with query will reset all cached documents for this collection.

        :param query: id, list of ids or query for documents to update.
        :type query: dict or list of basestring or tuple of basestring or basestring or bson.ObjectID
        :param update: update specification
        :type update: dict
        """
        #TODO: drop document from cache, if necessary
        if isinstance(query, dict):
            multi = True
        elif isinstance(query, (list, set,)):
            multi = True
            query = {'_id': {'$in': query}}
        elif isinstance(query, basestring):
            query = {'_id': query}
            multi = False
        else:
            raise TypeError('Invalid query: {}'.format(query))
        kwargs.setdefault('multi', multi)
        self._collection.update(query, update, safe=True, **kwargs)

    def remove(self, q=None):
        if q is not None:
            if isinstance(q, basestring):
                self._drop_cache_entry(q)
                q = {'_id': q}
            self._collection.remove(q)
        else:
            self._collection.remove()
            if self.use_cache:
                self._cache.clear()

    # Mapping implementation
    def __getitem__(self, item):
        return self.get(item)

    def __len__(self):
        return self._collection.count()

    def __iter__(self):
        return self.all()

    def keys(self):
        return self.ids()

class Proxy(MappingView):
    db = ''
    collection = ''
    use_cache = True
    include_fields = ()
    exclude_fields = ()
    dump_fields = ()

    _all = []
    _data_provider = None

    def __init__(self, _id=None, data=None):
        if _id is not None:
            data = self.get_document(_id)
        data = data or {}
        super(Proxy, self).__init__(data)

    @classmethod
    def find(cls,*args,**kwargs):
        return cls.get_data_provider().find(*args,**kwargs)

    @classmethod
    def get_data_provider(cls):
        if cls._data_provider is None:
            cls._data_provider = DataProvider(cls.db, cls.collection, cls.use_cache)
        return cls._data_provider

    @classmethod
    def get_document(cls, _id):
        document = cls.get_data_provider().get(
            _id, include_fields=cls.include_fields, exclude_fields=cls.exclude_fields)
        if not document:
            raise KeyError('Document "{}" not found'.format(_id))
        return cls.get_data_provider().get(_id)

    @classmethod
    def all(cls):
        for data in cls.get_data_provider().find(
                include_fields=cls.include_fields, exclude_fields=cls.exclude_fields):
            yield cls(data=data)

    @classmethod
    def ids(cls):
        return cls.get_data_provider().ids()

    @classmethod
    def get_names(cls):
        """
        Alias for ids
        """
        return cls.ids()

    _dump = None

    def dump(self):
        fields = (set(self.dump_fields) or set(self)) - set(self.exclude_fields)
        return dict((field, dump_value(self[field])) for field in fields)

    @property
    def id(self):
        return self['_id']

class DiffedProxy(Proxy, MutableMapping):
    def __init__(self, _id=None, data=None, diffs=()):
        super(DiffedProxy, self).__init__(_id=_id, data=data)
        self._data = Diffed(self._data)
        self.add_diff(*diffs)

    @property
    def diffs(self):
        return self._data.diffs

    def add_diff(self, *diffs):
        self._data.add_diff(*diffs)

    def remove_diff(self, *diffs):
        self._data.remove_diff(*diffs)

    def apply_diff(self, *diffs):
        self._data.apply_diff(*diffs)

    def flatten_diff(self, *diffs):
        self._data.flatten_diff(*diffs)

    def flatten(self):
        self._data.flatten()

    def get_current_diff(self):
        return self._data.get_current_diff()

    def dump(self):
        return self._data.dump()

    def dump_diff(self):
        return self._data.dump_diff()

    def __setitem__(self, item, value):
        self._data[item] = value

    def __delitem__(self, key):
        del self._data[key]

    def __getitem__(self, item):
        return self._data[item]
<commit_msg>Just a little proxy fix<commit_after>from collections import MutableMapping
from pymongo import MongoClient, Connection
from bson import ObjectId
from collections import Mapping
from cherrycommon.dictutils import MappingView, dump_value, Diffed
from cherrycommon.timeutils import seconds

DEFAULT_HOST = 'localhost'
DEFAULT_PORT = 27017
DEFAULT_DB_VERSION = 1

_mongo_clients = {}

def get_mongo_client(host=DEFAULT_HOST, port=DEFAULT_PORT, **kwargs):
    """Return (cached) MongoClient instance for provided URL.

    :param host: Host to connect.
    :type host: basestring
    :param port: Port to connect.
    :type port: int
    :rtype: MongoClient
    :return: Instance of MongoClient for provided host - port pair.
    """
    url = '{}:{}'.format(host, port)
    try:
        return _mongo_clients[(host, port)]
    except KeyError:
        client = _mongo_clients[(host, port)] = MongoClient(host, port, **kwargs)
        return client

class DataProvider(Mapping):
    def _get_collection(self, host, port, db, collection):
        try:
            return self._collection
        except AttributeError:
            self._collection = get_mongo_client(host, port)[db][collection]
            return self._collection

    @property
    def collection(self):
        return self._collection

    _global_cache = {}

    @classmethod
    def _get_cache(cls, db, collection):
        return cls._global_cache.setdefault((db, collection), {})

    _index_ttl = seconds(hours=1)

    def __init__(self, db, collection, use_cache=False, indexes=None, host=DEFAULT_HOST, port=DEFAULT_PORT):
        self._db_name = db
        self._collection_name = collection
        self._host = host
        self._port = port
        self._collection = self._get_collection(host, port, db, collection)

        if indexes is not None:
            for index_name in indexes:
                self._collection.ensure_index(index_name, ttl=self._index_ttl)

        if use_cache:
            self._cache = self._get_cache(db, collection)
        else:
            self._cache = None

    def _drop_cache_entry(self, pk):
        if self._cache:
            try:
                del self._cache[pk]
            except KeyError:
                pass

    def _prepare_fields(self, include_fields, exclude_fields):
        if self._cache:
            return None
        if include_fields:
            fields = dict.fromkeys(include_fields, 1)
            return fields
        if exclude_fields:
            return dict.fromkeys(exclude_fields, 0)

    @property
    def use_cache(self):
        return self._cache is not None

    def get(self, _id, include_fields=None, exclude_fields=None, force_reload=False):
        """
        Get document from collection by its primary key. 'fields' argument does not matter,
        if DataProvider caches it's data.
        """
        fields = self._prepare_fields(include_fields, exclude_fields)
        if self.use_cache and (not force_reload):
            try:
                return self._cache[_id]
            except KeyError:
                pass
        document = self._collection.find_one(_id, fields=fields)
        if self.use_cache:
            if document:
                self._cache[_id] = document
            else:
                self._drop_cache_entry(_id)
        return document

    @staticmethod
    def _keys_iterator(cursor):
        for document in cursor:
            yield document['_id'], document

    def find(self, *args, **kwargs):
        """Searches collection for documents, that matches filters. Cache is ignored for this operation.
        """
        include_fields = kwargs.pop('include_fields', {})
        exclude_fields = kwargs.pop('exclude_fields', {})
        kwargs['fields'] = kwargs.get('fields') or self._prepare_fields(include_fields, exclude_fields)
        keys = kwargs.pop('keys', False)
        cursor = self._collection.find(*args, **kwargs)
        if keys:
            return self._keys_iterator(cursor)
        else:
            return cursor

    def find_and_modify(self, *args, **kwargs):
        """Executes find and modify against the collection.
        """
        return self._collection.find_and_modify(*args, **kwargs)

    def find_one(self, spec, *args, **kwargs):
        include_fields = kwargs.pop('include_fields', {})
        exclude_fields = kwargs.pop('exclude_fields', {})
        kwargs['fields'] = kwargs.get('fields') or self._prepare_fields(include_fields, exclude_fields)
        if not isinstance(spec, dict):
            spec = {'_id': spec}
        return self._collection.find_one(spec, *args, **kwargs)

    def all(self, include_fields=None, exclude_fields=None, keys=False, *args, **kwargs):
        """Return cursor with all documents in collection.

        :rtype: Cursor
        """
        return self.find(include_fields=include_fields, exclude_fields=exclude_fields, keys=keys, *args, **kwargs)

    def ids(self):
        """List ids of all documents in collection
        """
        return self._collection.distinct('_id')

    def save(self, document, safe=False):
        """Saves document in collection. Creates one, if not exists yet.
        """
        self._collection.save(document, safe=safe)
        if hasattr(document, '_id'):
            self._drop_cache_entry(document['_id'])

    def insert(self, documents, **kwargs):
        """Stores documents into the collection.

        :param documents: document or list of documents to store.
        """
        if not isinstance(documents, list):
            documents = [documents]
        self._collection.insert(documents, **kwargs)

    def update(self, query, update, **kwargs):
        """Updates documents in collection.
        You can also pass named args, supported by pymongo.Collection.update method.
        Warning! Update with query will reset all cached documents for this collection.

        :param query: id, list of ids or query for documents to update.
        :type query: dict or list of basestring or tuple of basestring or basestring or bson.ObjectID
        :param update: update specification
        :type update: dict
        """
        #TODO: drop document from cache, if necessary
        if isinstance(query, dict):
            multi = True
        elif isinstance(query, (list, set,)):
            multi = True
            query = {'_id': {'$in': query}}
        elif isinstance(query, basestring):
            query = {'_id': query}
            multi = False
        else:
            raise TypeError('Invalid query: {}'.format(query))
        kwargs.setdefault('multi', multi)
        self._collection.update(query, update, safe=True, **kwargs)

    def remove(self, q=None):
        if q is not None:
            if isinstance(q, basestring):
                self._drop_cache_entry(q)
                q = {'_id': q}
            self._collection.remove(q)
        else:
            self._collection.remove()
            if self.use_cache:
                self._cache.clear()

    # Mapping implementation
    def __getitem__(self, item):
        return self.get(item)

    def __len__(self):
        return self._collection.count()

    def __iter__(self):
        return self.all()

    def keys(self):
        return self.ids()

class Proxy(MappingView):
    db = ''
    collection = ''
    use_cache = True
    include_fields = ()
    exclude_fields = ()
    dump_fields = ()

    _all = []
    _data_provider = None

    def __init__(self, _id=None, data=None):
        if _id is not None:
            data = self.get_document(_id)
        data = data or {}
        super(Proxy, self).__init__(data)

    @classmethod
    def find(cls,*args,**kwargs):
        return cls.get_data_provider().find(*args,**kwargs)

    @classmethod
    def get_data_provider(cls):
        if cls._data_provider is None:
            cls._data_provider = DataProvider(cls.db, cls.collection, cls.use_cache)
        return cls._data_provider

    @classmethod
    def get_document(cls, _id):
        document = cls.get_data_provider().get(
            _id, include_fields=cls.include_fields, exclude_fields=cls.exclude_fields)
        if not document:
            raise KeyError('Document "{}" not found'.format(_id))
        return document

    @classmethod
    def all(cls):
        for data in cls.get_data_provider().find(
                include_fields=cls.include_fields, exclude_fields=cls.exclude_fields):
            yield cls(data=data)

    @classmethod
    def ids(cls):
        return cls.get_data_provider().ids()

    @classmethod
    def get_names(cls):
        """
        Alias for ids
        """
        return cls.ids()

    _dump = None

    def dump(self):
        fields = (set(self.dump_fields) or set(self)) - set(self.exclude_fields)
        return dict((field, dump_value(self[field])) for field in fields)

    @property
    def id(self):
        return self['_id']

class DiffedProxy(Proxy, MutableMapping):
    def __init__(self, _id=None, data=None, diffs=()):
        super(DiffedProxy, self).__init__(_id=_id, data=data)
        self._data = Diffed(self._data)
        self.add_diff(*diffs)

    @property
    def diffs(self):
        return self._data.diffs

    def add_diff(self, *diffs):
        self._data.add_diff(*diffs)

    def remove_diff(self, *diffs):
        self._data.remove_diff(*diffs)

    def apply_diff(self, *diffs):
        self._data.apply_diff(*diffs)

    def flatten_diff(self, *diffs):
        self._data.flatten_diff(*diffs)

    def flatten(self):
        self._data.flatten()

    def get_current_diff(self):
        return self._data.get_current_diff()

    def dump(self):
        return self._data.dump()

    def dump_diff(self):
        return self._data.dump_diff()

    def __setitem__(self, item, value):
        self._data[item] = value

    def __delitem__(self, key):
        del self._data[key]

    def __getitem__(self, item):
        return self._data[item]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Test database compression functions
'''
# Import sorbic libs
import sorbic.db

# Import python libs
import os
import shutil
import unittest
import tempfile

class TestCompress(unittest.TestCase):
    '''
    Cover compression possibilities
    '''
    def test_compress_no_changes(self):
        '''
        Run a scale db execution with the given db kwargs
        '''
        entries = 100
        w_dir = tempfile.mkdtemp()
        root = os.path.join(w_dir, 'db_root')
        db = sorbic.db.DB(root)
        data = {1:1}
        for num in xrange(entries):
            key = str(num)
            db.insert(key, data)
        db.compress('', 0)
        for num in xrange(entries):
            key = str(num)
            pull_data = db.get(key)
            self.assertEqual(data, pull_data)
        shutil.rmtree(w_dir)

    def test_compress_no_changes_depth(self):
        '''
        Run a scale db execution with the given db kwargs
        '''
        entries = 100
        w_dir = tempfile.mkdtemp()
        root = os.path.join(w_dir, 'db_root')
        db = sorbic.db.DB(root)
        data = {1:1}
        key = 'foo/bar'
        for num in xrange(entries):
            db.insert(key, data)
        db.compress('foo', 0)
        for num in xrange(entries):
            pull_data = db.get(key)
            self.assertEqual(data, pull_data)
        shutil.rmtree(w_dir)
<commit_msg>Add compress test for key loss<commit_after># -*- coding: utf-8 -*-
'''
Test database compression functions
'''
# Import sorbic libs
import sorbic.db

# Import python libs
import os
import random
import shutil
import unittest
import tempfile

class TestCompress(unittest.TestCase):
    '''
    Cover compression possibilities
    '''
    def test_compress_no_changes(self):
        '''
        Run a scale db execution with the given db kwargs
        '''
        entries = 100
        w_dir = tempfile.mkdtemp()
        root = os.path.join(w_dir, 'db_root')
        db = sorbic.db.DB(root)
        data = {1:1}
        for num in xrange(entries):
            key = str(num)
            db.insert(key, data)
        db.compress('', 0)
        for num in xrange(entries):
            key = str(num)
            pull_data = db.get(key)
            self.assertEqual(data, pull_data)
        shutil.rmtree(w_dir)

    def test_compress_no_changes_depth(self):
        '''
        Run a scale db execution with the given db kwargs
        '''
        entries = 100
        w_dir = tempfile.mkdtemp()
        root = os.path.join(w_dir, 'db_root')
        db = sorbic.db.DB(root)
        data = {1:1}
        key = 'foo/bar'
        for num in xrange(entries):
            db.insert(key, data)
        db.compress('foo', 0)
        for num in xrange(entries):
            pull_data = db.get(key)
            self.assertEqual(data, pull_data)
        shutil.rmtree(w_dir)

    def test_compress_changes(self):
        '''
        Compress a db with removed keys
        '''
        entries = 100
        w_dir = tempfile.mkdtemp()
        root = os.path.join(w_dir, 'db_root')
        db = sorbic.db.DB(root)
        rands = set()
        data = {1:1}
        for num in xrange(entries):
            key = str(num)
            db.insert(key, data)
        for _ in xrange(entries):
            rands.add(random.randint(0, entries))
        for key in rands:
            db.rm(str(key))
        db.compress('', 0)
        for num in xrange(entries):
            key = str(num)
            pull_data = db.get(key)
            if num in rands:
                self.assertIsNone(pull_data)
            else:
                self.assertEqual(data, pull_data)
        shutil.rmtree(w_dir)
<|endoftext|>


<commit_before># Licensed to the Software Freedom Conservancy (SFC) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The SFC licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import base64
import os
import unittest
import zipfile

try:
    from io import BytesIO
except ImportError:
    from cStringIO import StringIO as BytesIO

try:
    unicode
except NameError:
    unicode = str

from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy, ProxyType
from selenium.test.selenium.webdriver.common.webserver import SimpleWebServer

class TestFirefoxProfile:

    def setup_method(self, method):
        self.driver = webdriver.Firefox()
        self.webserver = SimpleWebServer()
        self.webserver.start()

    def test_that_we_can_accept_a_profile(self):
        profile1 = webdriver.FirefoxProfile()
        profile1.set_preference("startup.homepage_welcome_url",
            self.webserver.where_is('simpleTest.html'))
        profile1.update_preferences()

        profile2 = webdriver.FirefoxProfile(profile1.path)
        driver = webdriver.Firefox(firefox_profile=profile2)
        title = driver.title
        driver.quit()
        assert "Hello WebDriver" == title

    def test_that_prefs_are_written_in_the_correct_format(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        profile = webdriver.FirefoxProfile()
        profile.set_preference("sample.preference", "hi there")
        profile.update_preferences()

        assert 'hi there' == profile.default_preferences["sample.preference"]

        encoded = profile.encoded
        decoded = base64.decodestring(encoded)
        fp = BytesIO(decoded)
        zip = zipfile.ZipFile(fp, "r")
        for entry in zip.namelist():
            if entry.endswith("user.js"):
                user_js = zip.read(entry)
                for line in user_js.splitlines():
                    if line.startswith(b'user_pref("sample.preference",'):
                        assert True == line.endswith(b'hi there");')
            # there should be only one user.js
            break
        fp.close()

    def test_that_unicode_prefs_are_written_in_the_correct_format(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        profile = webdriver.FirefoxProfile()
        profile.set_preference('sample.preference.2', unicode('hi there'))
        profile.update_preferences()

        assert 'hi there' == profile.default_preferences["sample.preference.2"]

        encoded = profile.encoded
        decoded = base64.decodestring(encoded)
        fp = BytesIO(decoded)
        zip = zipfile.ZipFile(fp, "r")
        for entry in zip.namelist():
            if entry.endswith('user.js'):
                user_js = zip.read(entry)
                for line in user_js.splitlines():
                    if line.startswith(b'user_pref("sample.preference.2",'):
                        assert True == line.endswith(b'hi there");')
            # there should be only one user.js
            break
        fp.close()

    def test_that_integer_prefs_are_written_in_the_correct_format(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        profile = webdriver.FirefoxProfile()
        profile.set_preference("sample.int.preference", 12345)
        profile.update_preferences()
        assert 12345 == profile.default_preferences["sample.int.preference"]

    def test_that_boolean_prefs_are_written_in_the_correct_format(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        profile = webdriver.FirefoxProfile()
        profile.set_preference("sample.bool.preference", True)
        profile.update_preferences()
        assert True == profile.default_preferences["sample.bool.preference"]

    def test_that_we_delete_the_profile(self):
        path = self.driver.firefox_profile.path
        self.driver.quit()
        assert not os.path.exists(path)

    def test_profiles_do_not_share_preferences(self):
        self.profile1 = webdriver.FirefoxProfile()
        self.profile1.accept_untrusted_certs = False
        self.profile2 = webdriver.FirefoxProfile()
        # Default is true. Should remain so.
        assert self.profile2.default_preferences["webdriver_accept_untrusted_certs"] == True

    def test_none_proxy_is_set(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        self.profile = webdriver.FirefoxProfile()
        proxy = None

        try:
            self.profile.set_proxy(proxy)
            assert False, "exception after passing empty proxy is expected"
        except ValueError as e:
            pass

        assert "network.proxy.type" not in self.profile.default_preferences

    def test_unspecified_proxy_is_set(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        self.profile = webdriver.FirefoxProfile()
        proxy = Proxy()

        self.profile.set_proxy(proxy)

        assert "network.proxy.type" not in self.profile.default_preferences

    def test_manual_proxy_is_set_in_profile(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        self.profile = webdriver.FirefoxProfile()
        proxy = Proxy()
        proxy.no_proxy = 'localhost, foo.localhost'
        proxy.http_proxy = 'some.url:1234'
        proxy.ftp_proxy = None
        proxy.sslProxy = 'some2.url'

        self.profile.set_proxy(proxy)

        assert self.profile.default_preferences["network.proxy.type"] == ProxyType.MANUAL['ff_value']
        assert self.profile.default_preferences["network.proxy.no_proxies_on"] == 'localhost, foo.localhost'
        assert self.profile.default_preferences["network.proxy.http"] == 'some.url'
        assert self.profile.default_preferences["network.proxy.http_port"] == 1234
        assert self.profile.default_preferences["network.proxy.ssl"] == 'some2.url'
        assert "network.proxy.ssl_port" not in self.profile.default_preferences
        assert "network.proxy.ftp" not in self.profile.default_preferences

    def test_pac_proxy_is_set_in_profile(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        self.profile = webdriver.FirefoxProfile()
        proxy = Proxy()
        proxy.proxy_autoconfig_url = 'http://some.url:12345/path'

        self.profile.set_proxy(proxy)

        assert self.profile.default_preferences["network.proxy.type"] == ProxyType.PAC['ff_value']
        assert self.profile.default_preferences["network.proxy.autoconfig_url"] == 'http://some.url:12345/path'

    def test_autodetect_proxy_is_set_in_profile(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        self.profile = webdriver.FirefoxProfile()
        proxy = Proxy()
        proxy.auto_detect = True

        self.profile.set_proxy(proxy)

        assert self.profile.default_preferences["network.proxy.type"] == ProxyType.AUTODETECT['ff_value']

    def teardown_method(self, method):
        try:
            self.driver.quit()
        except:
            pass #don't care since we may have killed the browser above
        self.webserver.stop()

    def _pageURL(self, name):
        return self.webserver.where_is(name + '.html')

    def _loadSimplePage(self):
        self._loadPage("simpleTest")

    def _loadPage(self, name):
        self.driver.get(self._pageURL(name))

def teardown_module(module):
    try:
        TestFirefoxProfile.driver.quit()
    except:
        pass #Don't Care since we may have killed the browser above
<commit_msg>Allow homepage override in Firefox profile test<commit_after># Licensed to the Software Freedom Conservancy (SFC) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The SFC licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import base64
import os
import unittest
import zipfile

try:
    from io import BytesIO
except ImportError:
    from cStringIO import StringIO as BytesIO

try:
    unicode
except NameError:
    unicode = str

from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy, ProxyType
from selenium.test.selenium.webdriver.common.webserver import SimpleWebServer

class TestFirefoxProfile:

    def setup_method(self, method):
        self.driver = webdriver.Firefox()
        self.webserver = SimpleWebServer()
        self.webserver.start()

    def test_that_we_can_accept_a_profile(self):
        profile1 = webdriver.FirefoxProfile()
        profile1.set_preference("browser.startup.homepage_override.mstone", "")
        profile1.set_preference("startup.homepage_welcome_url",
            self.webserver.where_is('simpleTest.html'))
        profile1.update_preferences()

        profile2 = webdriver.FirefoxProfile(profile1.path)
        driver = webdriver.Firefox(firefox_profile=profile2)
        title = driver.title
        driver.quit()
        assert "Hello WebDriver" == title

    def test_that_prefs_are_written_in_the_correct_format(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        profile = webdriver.FirefoxProfile()
        profile.set_preference("sample.preference", "hi there")
        profile.update_preferences()

        assert 'hi there' == profile.default_preferences["sample.preference"]

        encoded = profile.encoded
        decoded = base64.decodestring(encoded)
        fp = BytesIO(decoded)
        zip = zipfile.ZipFile(fp, "r")
        for entry in zip.namelist():
            if entry.endswith("user.js"):
                user_js = zip.read(entry)
                for line in user_js.splitlines():
                    if line.startswith(b'user_pref("sample.preference",'):
                        assert True == line.endswith(b'hi there");')
            # there should be only one user.js
            break
        fp.close()

    def test_that_unicode_prefs_are_written_in_the_correct_format(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        profile = webdriver.FirefoxProfile()
        profile.set_preference('sample.preference.2', unicode('hi there'))
        profile.update_preferences()

        assert 'hi there' == profile.default_preferences["sample.preference.2"]

        encoded = profile.encoded
        decoded = base64.decodestring(encoded)
        fp = BytesIO(decoded)
        zip = zipfile.ZipFile(fp, "r")
        for entry in zip.namelist():
            if entry.endswith('user.js'):
                user_js = zip.read(entry)
                for line in user_js.splitlines():
                    if line.startswith(b'user_pref("sample.preference.2",'):
                        assert True == line.endswith(b'hi there");')
            # there should be only one user.js
            break
        fp.close()

    def test_that_integer_prefs_are_written_in_the_correct_format(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        profile = webdriver.FirefoxProfile()
        profile.set_preference("sample.int.preference", 12345)
        profile.update_preferences()
        assert 12345 == profile.default_preferences["sample.int.preference"]

    def test_that_boolean_prefs_are_written_in_the_correct_format(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        profile = webdriver.FirefoxProfile()
        profile.set_preference("sample.bool.preference", True)
        profile.update_preferences()
        assert True == profile.default_preferences["sample.bool.preference"]

    def test_that_we_delete_the_profile(self):
        path = self.driver.firefox_profile.path
        self.driver.quit()
        assert not os.path.exists(path)

    def test_profiles_do_not_share_preferences(self):
        self.profile1 = webdriver.FirefoxProfile()
        self.profile1.accept_untrusted_certs = False
        self.profile2 = webdriver.FirefoxProfile()
        # Default is true. Should remain so.
        assert self.profile2.default_preferences["webdriver_accept_untrusted_certs"] == True

    def test_none_proxy_is_set(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        self.profile = webdriver.FirefoxProfile()
        proxy = None

        try:
            self.profile.set_proxy(proxy)
            assert False, "exception after passing empty proxy is expected"
        except ValueError as e:
            pass

        assert "network.proxy.type" not in self.profile.default_preferences

    def test_unspecified_proxy_is_set(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        self.profile = webdriver.FirefoxProfile()
        proxy = Proxy()

        self.profile.set_proxy(proxy)

        assert "network.proxy.type" not in self.profile.default_preferences

    def test_manual_proxy_is_set_in_profile(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        self.profile = webdriver.FirefoxProfile()
        proxy = Proxy()
        proxy.no_proxy = 'localhost, foo.localhost'
        proxy.http_proxy = 'some.url:1234'
        proxy.ftp_proxy = None
        proxy.sslProxy = 'some2.url'

        self.profile.set_proxy(proxy)

        assert self.profile.default_preferences["network.proxy.type"] == ProxyType.MANUAL['ff_value']
        assert self.profile.default_preferences["network.proxy.no_proxies_on"] == 'localhost, foo.localhost'
        assert self.profile.default_preferences["network.proxy.http"] == 'some.url'
        assert self.profile.default_preferences["network.proxy.http_port"] == 1234
        assert self.profile.default_preferences["network.proxy.ssl"] == 'some2.url'
        assert "network.proxy.ssl_port" not in self.profile.default_preferences
        assert "network.proxy.ftp" not in self.profile.default_preferences

    def test_pac_proxy_is_set_in_profile(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        self.profile = webdriver.FirefoxProfile()
        proxy = Proxy()
        proxy.proxy_autoconfig_url = 'http://some.url:12345/path'

        self.profile.set_proxy(proxy)

        assert self.profile.default_preferences["network.proxy.type"] == ProxyType.PAC['ff_value']
        assert self.profile.default_preferences["network.proxy.autoconfig_url"] == 'http://some.url:12345/path'

    def test_autodetect_proxy_is_set_in_profile(self):
        # The setup gave us a browser but we dont need it
        self.driver.quit()

        self.profile = webdriver.FirefoxProfile()
        proxy = Proxy()
        proxy.auto_detect = True

        self.profile.set_proxy(proxy)

        assert self.profile.default_preferences["network.proxy.type"] == ProxyType.AUTODETECT['ff_value']

    def teardown_method(self, method):
        try:
            self.driver.quit()
        except:
            pass #don't care since we may have killed the browser above
        self.webserver.stop()

    def _pageURL(self, name):
        return self.webserver.where_is(name + '.html')

    def _loadSimplePage(self):
        self._loadPage("simpleTest")

    def _loadPage(self, name):
        self.driver.get(self._pageURL(name))

def teardown_module(module):
    try:
        TestFirefoxProfile.driver.quit()
    except:
        pass #Don't Care since we may have killed the browser above
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#

# Import Libraries
from telegram.ext import Updater, CommandHandler, MessageHandler, Filters
import logging
import requests
import json
import os

# Enable logging
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
                    level=logging.INFO)

logger = logging.getLogger(__name__)

"""Response
weather - Get Latest Weather Report
psi - Get Latest PSI Report
"""

def psi3hour(bot, update):

    # Make the HTTP request.
    DATAGOV = str(os.environ.get('DATAGOV'))
    headers = {'api-key': DATAGOV}
    r = requests.get('https://api.data.gov.sg/v1/environment/psi', headers=headers)

    # Load data into Dictionary and get reading
    data = json.loads(r.text)
    hourly = data['items'][0]['readings']['psi_three_hourly']
    timestampp = data['items'][0]['timestamp'][:19].replace("T"," ")

    # Create Response
    final_string = "The 3 hourly PSI Reading at " + timestampp + " is actually \n\n"
    for key in sorted(hourly):
        final_string  =  final_string + (str(key) + " " + str(hourly[key]) + "\n")
    bot.sendMessage(update.message.chat_id, text=final_string)

def weathernow(bot, update):
    DATAGOV = str(os.environ.get('DATAGOV'))
    headers = {'api-key': DATAGOV}
    r = requests.get('https://api.data.gov.sg/v1/environment/24-hour-weather-forecast', headers=headers)

    # Load data into Dictionary and get reading
    data = json.loads(r.text)
    forecast = data['items'][0]['general']['forecast']
    h_ = data['items'][0]['general']['temperature']['high']
    l_ = data['items'][0]['general']['temperature']['low']

    # Create Response
    final_string = "In General the weather for next looks like " + forecast + " with a high of " + str(h_) + \
    "°C and a low of " + str(l_) + "°C\n\nForecast Next 12 Hrs\n\n"

    # Add 12 hr cast
    nowcast = data['items'][0]['periods'][0]['regions']
    for key in sorted(nowcast):
        final_string  =  final_string + (str(key) + " - " + str(nowcast[key]) + "\n")
    final_string = final_string + "\nForecast Tomorrow\n\n"

    # Add 24 hr cast
    nowcast = data['items'][0]['periods'][1]['regions']
    for key in sorted(nowcast):
        final_string  =  final_string + (str(key) + " - " + str(nowcast[key]) + "\n")

    bot.sendMessage(update.message.chat_id, text=final_string)

def start(bot, update):
    bot.sendMessage(update.message.chat_id, text='Hello! I am @ShiokBot! \nThe helpful singlish spouting bot! \n\nAvailable Commands \n/psi - Report the latest PSI readings lo \n/weather - Report the latest weather lah')

def help(bot, update):
    bot.sendMessage(update.message.chat_id, text='Hello! I am @ShiokBot! \nThe helpful singlish spouting bot! \n\nAvailable Commands \n/psi - Report the latest PSI readings lo \n/weather - Report the latest weather lah')

def error(bot, update, error):
    logger.warn('Update "%s" caused error "%s"' % (update, error))

def main():
    # Create the EventHandler and pass it your bot's token.
    TELEGRAM = str(os.environ.get('TELEGRAM'))
    updater = Updater(TELEGRAM)

    # Get the dispatcher to register handlers
    dp = updater.dispatcher

    # on different commands - answer in Telegram
    dp.add_handler(CommandHandler("start", start))
    dp.add_handler(CommandHandler("help", help))
    dp.add_handler(CommandHandler("psi", psi3hour))
    dp.add_handler(CommandHandler("weather", weathernow))

    # log all errors
    dp.add_error_handler(error)

    # Start the Bot
    """
    #DEV
    updater.start_polling()

    # Run the bot until the you presses Ctrl-C or the process receives SIGINT,
    # SIGTERM or SIGABRT. This should be used most of the time, since
    # start_polling() is non-blocking and will stop the bot gracefully.
    updater.idle()
    """

    #PROD
    PORT = int(os.environ.get('PORT', '5000'))
    updater.start_webhook(listen="0.0.0.0",
                          port=PORT,
                          url_path=TELEGRAM)
    updater.bot.setWebhook("https://shiokbot.herokuapp.com/" + TELEGRAM)
    updater.idle()

if __name__ == '__main__':
    main()
<commit_msg>edit #change request<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#

# Import Libraries
from telegram.ext import Updater, CommandHandler, MessageHandler, Filters
import logging
import requests
import json
import os

# Enable logging
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
                    level=logging.INFO)

logger = logging.getLogger(__name__)

"""Response
weather - Get Latest Weather Report
psi - Get Latest PSI Report
"""

def psi3hour(bot, update):

    # Make the HTTP request.
    DATAGOV = str(os.environ.get('DATAGOV'))
    headers = {'api-key': DATAGOV}
    r = requests.get('https://api.data.gov.sg/v1/environment/psi', headers=headers)

    # Load data into Dictionary and get reading
    data = json.loads(r.text)
    hourly = data['items'][0]['readings']['psi_three_hourly']
    timestampp = data['items'][0]['timestamp'][:19].replace("T"," ")

    # Create Response
    final_string = "The 3 hourly PSI Reading at " + timestampp + " is actually \n\n"
    for key in sorted(hourly):
        final_string  =  final_string + (str(key) + " " + str(hourly[key]) + "\n")
    bot.sendMessage(update.message.chat_id, text=final_string)

def weathernow(bot, update):
    DATAGOV = str(os.environ.get('DATAGOV'))
    headers = {'api-key': DATAGOV}
    r = requests.get('https://api.data.gov.sg/v1/environment/24-hour-weather-forecast', headers=headers)

    # Load data into Dictionary and get reading
    data = json.loads(r.text)
    forecast = data['items'][0]['general']['forecast']
    h_ = data['items'][0]['general']['temperature']['high']
    l_ = data['items'][0]['general']['temperature']['low']

    # Create Response
    final_string = "In General the weather looks like " + forecast + " with a high of " + str(h_) + \
    "°C and a low of " + str(l_) + "°C\n\nForecast Next 12 Hrs\n\n"

    # Add 12 hr cast
    nowcast = data['items'][0]['periods'][0]['regions']
    for key in sorted(nowcast):
        final_string  =  final_string + (str(key) + " - " + str(nowcast[key]) + "\n")
    final_string = final_string + "\nForecast Tomorrow\n\n"

    # Add 24 hr cast
    nowcast = data['items'][0]['periods'][1]['regions']
    for key in sorted(nowcast):
        final_string  =  final_string + (str(key) + " - " + str(nowcast[key]) + "\n")

    bot.sendMessage(update.message.chat_id, text=final_string)

def start(bot, update):
    bot.sendMessage(update.message.chat_id, text='Hello! I am @ShiokBot! \nThe helpful singlish spouting bot! \n\nAvailable Commands \n/psi - Report the latest PSI readings lo \n/weather - Report the latest weather lah')

def help(bot, update):
    bot.sendMessage(update.message.chat_id, text='Hello! I am @ShiokBot! \nThe helpful singlish spouting bot! \n\nAvailable Commands \n/psi - Report the latest PSI readings lo \n/weather - Report the latest weather lah')

def error(bot, update, error):
    logger.warn('Update "%s" caused error "%s"' % (update, error))

def main():
    # Create the EventHandler and pass it your bot's token.
    TELEGRAM = str(os.environ.get('TELEGRAM'))
    updater = Updater(TELEGRAM)

    # Get the dispatcher to register handlers
    dp = updater.dispatcher

    # on different commands - answer in Telegram
    dp.add_handler(CommandHandler("start", start))
    dp.add_handler(CommandHandler("help", help))
    dp.add_handler(CommandHandler("psi", psi3hour))
    dp.add_handler(CommandHandler("weather", weathernow))

    # log all errors
    dp.add_error_handler(error)

    # Start the Bot
    """
    #DEV
    updater.start_polling()

    # Run the bot until the you presses Ctrl-C or the process receives SIGINT,
    # SIGTERM or SIGABRT. This should be used most of the time, since
    # start_polling() is non-blocking and will stop the bot gracefully.
    updater.idle()
    """

    #PROD
    PORT = int(os.environ.get('PORT', '5000'))
    updater.start_webhook(listen="0.0.0.0",
                          port=PORT,
                          url_path=TELEGRAM)
    updater.bot.setWebhook("https://shiokbot.herokuapp.com/" + TELEGRAM)
    updater.idle()

if __name__ == '__main__':
    main()
<|endoftext|>