<commit_before>import sha
import random

from django.conf import settings
from django.db import models
from django.template.loader import render_to_string
from django.db.models import permalink
from django.contrib.sites.models import Site
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _

RESERVED_NAMES = ('admin', )

class RegistrationManager(models.Manager):

    def send_welcome(self, registration):
        """Send the welcome email for a registration."""

        from django.core.mail import send_mail

        current_site = Site.objects.get_current()
        subject = render_to_string('registration/email/subject.txt',
                                   {'site':current_site}).strip()
        message = render_to_string('registration/email/welcome.txt',
                                   {'site':current_site,
                                    'registration':registration})

        send_mail(subject, message, settings.DEFAULT_FROM_EMAIL,
                  [registration.email])

    def create_registration(self, transaction_id, email, last, first,
                            send_email = True):
        registration = PartialRegistration(
            transaction_id = transaction_id,
            email = email,
            last_name = last,
            first_name = first)

        registration.save()

        if send_email:
            self.send_welcome(registration)

        return registration

class PartialRegistration(models.Model):

    key = models.CharField(max_length=40, primary_key=True,
                           editable=False)

    last_name = models.CharField(max_length=30)
    first_name = models.CharField(max_length=30)
    email = models.EmailField()
    complete = models.BooleanField(default=False)
    transaction_id = models.CharField(max_length=255, blank=True, null=True)

    user = models.ForeignKey(User, blank=True, null=True)

    def __unicode__(self):
        return u"%s (%s, %s)" % (
            self.email, self.last_name, self.first_name)

    @permalink
    def get_absolute_url(self):
        return ('commoner.registration.views.complete', (self.key,))

    def save(self):

        # see if we already have a key
        if not(self.key):
            # set the SHA-1 hash
            salt = sha.new(str(random.random())).hexdigest()[:5]
            self.key = sha.new(salt+self.email+self.last_name).hexdigest()

        # call the real save method
        super(PartialRegistration, self).save()

    objects = RegistrationManager()

    class Meta:
        verbose_name=_("User registration")
<commit_msg>Ponies are protected.<commit_after>import sha
import random

from django.conf import settings
from django.db import models
from django.template.loader import render_to_string
from django.db.models import permalink
from django.contrib.sites.models import Site
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _

RESERVED_NAMES = ('admin', 'pony')

class RegistrationManager(models.Manager):

    def send_welcome(self, registration):
        """Send the welcome email for a registration."""

        from django.core.mail import send_mail

        current_site = Site.objects.get_current()
        subject = render_to_string('registration/email/subject.txt',
                                   {'site':current_site}).strip()
        message = render_to_string('registration/email/welcome.txt',
                                   {'site':current_site,
                                    'registration':registration})

        send_mail(subject, message, settings.DEFAULT_FROM_EMAIL,
                  [registration.email])

    def create_registration(self, transaction_id, email, last, first,
                            send_email = True):
        registration = PartialRegistration(
            transaction_id = transaction_id,
            email = email,
            last_name = last,
            first_name = first)

        registration.save()

        if send_email:
            self.send_welcome(registration)

        return registration

class PartialRegistration(models.Model):

    key = models.CharField(max_length=40, primary_key=True,
                           editable=False)

    last_name = models.CharField(max_length=30)
    first_name = models.CharField(max_length=30)
    email = models.EmailField()
    complete = models.BooleanField(default=False)
    transaction_id = models.CharField(max_length=255, blank=True, null=True)

    user = models.ForeignKey(User, blank=True, null=True)

    def __unicode__(self):
        return u"%s (%s, %s)" % (
            self.email, self.last_name, self.first_name)

    @permalink
    def get_absolute_url(self):
        return ('commoner.registration.views.complete', (self.key,))

    def save(self):

        # see if we already have a key
        if not(self.key):
            # set the SHA-1 hash
            salt = sha.new(str(random.random())).hexdigest()[:5]
            self.key = sha.new(salt+self.email+self.last_name).hexdigest()

        # call the real save method
        super(PartialRegistration, self).save()

    objects = RegistrationManager()

    class Meta:
        verbose_name=_("User registration")
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import sys
import os
from panda import Panda
from panda.python.uds import UdsClient, NegativeResponseError, DATA_IDENTIFIER_TYPE

if __name__ == "__main__":
  address = 0x18da30f1 # Honda EPS
  panda = Panda()
  uds_client = UdsClient(panda, address, debug=False)

  print("tester present ...")
  uds_client.tester_present()

  try:
    print("")
    print("read data by id: boot software id ...")
    data = uds_client.read_data_by_identifier(DATA_IDENTIFIER_TYPE.BOOT_SOFTWARE_IDENTIFICATION)
    print(data.decode('utf-8'))
  except NegativeResponseError as e:
    print(e)

  try:
    print("")
    print("read data by id: application software id ...")
    data = uds_client.read_data_by_identifier(DATA_IDENTIFIER_TYPE.APPLICATION_SOFTWARE_IDENTIFICATION)
    print(data.decode('utf-8'))
  except NegativeResponseError as e:
    print(e)

  try:
    print("")
    print("read data by id: application data id ...")
    data = uds_client.read_data_by_identifier(DATA_IDENTIFIER_TYPE.APPLICATION_DATA_IDENTIFICATION)
    print(data.decode('utf-8'))
  except NegativeResponseError as e:
    print(e)

  try:
    print("")
    print("read data by id: boot software fingerprint ...")
    data = uds_client.read_data_by_identifier(DATA_IDENTIFIER_TYPE.BOOT_SOFTWARE_FINGERPRINT)
    print(data.decode('utf-8'))
  except NegativeResponseError as e:
    print(e)

  try:
    print("")
    print("read data by id: application software fingerprint ...")
    data = uds_client.read_data_by_identifier(DATA_IDENTIFIER_TYPE.APPLICATION_SOFTWARE_FINGERPRINT)
    print(data.decode('utf-8'))
  except NegativeResponseError as e:
    print(e)

  try:
    print("")
    print("read data by id: application data fingerprint ...")
    data = uds_client.read_data_by_identifier(DATA_IDENTIFIER_TYPE.APPLICATION_DATA_FINGERPRINT)
    print(data.decode('utf-8'))
  except NegativeResponseError as e:
    print(e)
<commit_msg>fix linter 2<commit_after>#!/usr/bin/env python3
from panda import Panda
from panda.python.uds import UdsClient, NegativeResponseError, DATA_IDENTIFIER_TYPE

if __name__ == "__main__":
  address = 0x18da30f1 # Honda EPS
  panda = Panda()
  uds_client = UdsClient(panda, address, debug=False)

  print("tester present ...")
  uds_client.tester_present()

  try:
    print("")
    print("read data by id: boot software id ...")
    data = uds_client.read_data_by_identifier(DATA_IDENTIFIER_TYPE.BOOT_SOFTWARE_IDENTIFICATION)
    print(data.decode('utf-8'))
  except NegativeResponseError as e:
    print(e)

  try:
    print("")
    print("read data by id: application software id ...")
    data = uds_client.read_data_by_identifier(DATA_IDENTIFIER_TYPE.APPLICATION_SOFTWARE_IDENTIFICATION)
    print(data.decode('utf-8'))
  except NegativeResponseError as e:
    print(e)

  try:
    print("")
    print("read data by id: application data id ...")
    data = uds_client.read_data_by_identifier(DATA_IDENTIFIER_TYPE.APPLICATION_DATA_IDENTIFICATION)
    print(data.decode('utf-8'))
  except NegativeResponseError as e:
    print(e)

  try:
    print("")
    print("read data by id: boot software fingerprint ...")
    data = uds_client.read_data_by_identifier(DATA_IDENTIFIER_TYPE.BOOT_SOFTWARE_FINGERPRINT)
    print(data.decode('utf-8'))
  except NegativeResponseError as e:
    print(e)

  try:
    print("")
    print("read data by id: application software fingerprint ...")
    data = uds_client.read_data_by_identifier(DATA_IDENTIFIER_TYPE.APPLICATION_SOFTWARE_FINGERPRINT)
    print(data.decode('utf-8'))
  except NegativeResponseError as e:
    print(e)

  try:
    print("")
    print("read data by id: application data fingerprint ...")
    data = uds_client.read_data_by_identifier(DATA_IDENTIFIER_TYPE.APPLICATION_DATA_FINGERPRINT)
    print(data.decode('utf-8'))
  except NegativeResponseError as e:
    print(e)
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright 2017 Northern.tech AS
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        https://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

from fabric.api import *
import logging
import requests
from common_docker import stop_docker_compose, log_files
import random
import filelock
import uuid
import subprocess
import os
import re
import pytest
import distutils.spawn
import log

logging.getLogger("requests").setLevel(logging.CRITICAL)
logging.getLogger("paramiko").setLevel(logging.CRITICAL)
logging.getLogger("urllib3").setLevel(logging.CRITICAL)
logging.getLogger("filelock").setLevel(logging.INFO)
logger = log.setup_custom_logger("root", "master")

docker_compose_instance = "mender" + str(random.randint(0, 9999999))

docker_lock = filelock.FileLock("docker_lock")
production_setup_lock = filelock.FileLock(".exposed_ports_lock")

inline_logs = False
mt_docker_compose_file = ""
run_tenant_tests = True

try:
    requests.packages.urllib3.disable_warnings()
except:
    pass

def pytest_addoption(parser):
    parser.addoption("--clients", action="store", default="localhost:8080",
                     help="Comma-seperate mender hosts, example: 10.100.10.11:8822, 10.100.10.12:8822")
    parser.addoption("--gateway", action="store", default="127.0.0.1:8080",
                     help="Host of mender gateway")

    parser.addoption("--api", action="store", default="0.1", help="API version used in HTTP requests")
    parser.addoption("--image", action="store_true", default="core-image-full-cmdline-vexpress-qemu.ext4", help="Valid update image")
    parser.addoption("--runslow", action="store_true", help="run slow tests")
    parser.addoption("--runfast", action="store_true", help="run fast tests")
    parser.addoption("--runnightly", action="store_true", help="run nightly (very slow) tests")
    parser.addoption("--runs3", action="store_true", help="run fast tests")

    parser.addoption("--upgrade-from", action="store", help="perform upgrade test", default="")
    parser.addoption("--no-teardown", action="store_true", help="Don't tear down environment after tests are run")
    parser.addoption("--inline-logs", action="store_true", help="Don't redirect docker-compose logs to a file")

    parser.addoption("--mt-docker-compose-file", action="store", type=str, help="Docker-compose file that enables multi-tenancy (required for some tests)")

def pytest_configure(config):
    global extra_files, inline_logs, mt_docker_compose_file
    verify_sane_test_environment()

    env.api_version = config.getoption("api")
    env.valid_image = config.getoption("image")

    inline_logs = config.getoption("--inline-logs")

    if config.getoption("--mt-docker-compose-file") is None:
        if os.path.exists("../docker-compose.mt.yml"):
            logger.warn("--mt-docker-compose-file not set, but ../docker-compose.mt.yml exists, using that file.")
            mt_docker_compose_file = "-f ../docker-compose.mt.yml"
    else:
        mt_docker_compose_file = "-f %s" % config.getoption("--mt-docker-compose-file")

    env.password = ""

    # Bash not always available, nor currently required:
    env.shell = "/bin/sh -c"

    # Disable known_hosts file, to avoid "host identification changed" errors.
    env.disable_known_hosts = True

    env.abort_on_prompts = True
    # Don't allocate pseudo-TTY by default, since it is not fully functional.
    # It can still be overriden on a case by case basis by passing
    # "pty = True/False" to the various fabric functions. See
    # https://www.fabfile.org/faq.html about init scripts.
    env.always_use_pty = False

    # Don't combine stderr with stdout. The login profile sometimes prints
    # terminal specific codes there, and we don't want it interfering with our
    # output. It can still be turned on on a case by case basis by passing
    # combine_stderr to each run() or sudo() command.
    env.combine_stderr = False

    env.user = "root"

    env.connection_attempts = 50
    env.eagerly_disconnect = True
    env.banner_timeout = 10

    version = subprocess.check_output(["../extra/release_tool.py", "--version-of", "integration"])
    if re.match("^[0-9]+\.[0-9]+\.[x0-9]+.*", version):
        # Don't run tenant tests for release branches.
        global run_tenant_tests
        run_tenant_tests = False

def pytest_runtest_setup(item):
    logger = log.setup_custom_logger("root", item.name)
    logger.info("%s is starting.... " % item.name)

def pytest_exception_interact(node, call, report):
    if report.failed:
        for log in log_files:
            logger.info("printing content of : %s" % log)
            with open(log) as f:
                for line in f.readlines():
                    logger.info("%s: %s" % (log, line))

@pytest.mark.hookwrapper
def pytest_runtest_makereport(item, call):
    pytest_html = item.config.pluginmanager.getplugin('html')
    if pytest_html is None:
        yield
        return
    outcome = yield
    report = outcome.get_result()
    extra = getattr(report, 'extra', [])
    if report.failed:
        url = ""
        if os.getenv("UPLOAD_BACKEND_LOGS_ON_FAIL", False):
            if len(log_files) > 0:
                # we already have s3cmd configured on our build machine, so use it directly
                s3_object_name = str(uuid.uuid4()) + ".log"
                ret = subprocess.call("s3cmd put %s s3://mender-backend-logs/%s" % (log_files[-1], s3_object_name), shell=True)
                if int(ret) == 0:
                    url = "https://s3-eu-west-1.amazonaws.com/mender-backend-logs/" + s3_object_name
                else:
                    logger.warn("uploading backend logs failed.")
            else:
                logger.warn("no log files found, did the backend actually start?")
        else:
            logger.warn("not uploading backend log files because UPLOAD_BACKEND_LOGS_ON_FAIL not set")

        # always add url to report
        extra.append(pytest_html.extras.url(url))
        report.extra = extra

def pytest_unconfigure(config):
    if not config.getoption("--no-teardown"):
        stop_docker_compose()

    for log in log_files:
        try:
            os.remove(log)
        except:
            pass

def pytest_runtest_teardown(item, nextitem):
    if nextitem is None:
        stop_docker_compose()

def get_valid_image():
    return env.valid_image

def verify_sane_test_environment():
    # check if required tools are in PATH, add any other checks here
    if distutils.spawn.find_executable("mender-stress-test-client") is None:
        raise SystemExit("mender-stress-test-client not found in PATH")

    if distutils.spawn.find_executable("mender-artifact") is None:
        raise SystemExit("mender-artifact not found in PATH")

    if distutils.spawn.find_executable("docker") is None:
        raise SystemExit("docker not found in PATH")

    ret = subprocess.call("docker ps > /dev/null", shell=True)
    if ret != 0:
        raise SystemExit("not able to use docker, is your user part of the docker group?")
<commit_msg>tests: Fix version detection when remote branches are involved.<commit_after>#!/usr/bin/python
# Copyright 2017 Northern.tech AS
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        https://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

from fabric.api import *
import logging
import requests
from common_docker import stop_docker_compose, log_files
import random
import filelock
import uuid
import subprocess
import os
import re
import pytest
import distutils.spawn
import log

logging.getLogger("requests").setLevel(logging.CRITICAL)
logging.getLogger("paramiko").setLevel(logging.CRITICAL)
logging.getLogger("urllib3").setLevel(logging.CRITICAL)
logging.getLogger("filelock").setLevel(logging.INFO)
logger = log.setup_custom_logger("root", "master")

docker_compose_instance = "mender" + str(random.randint(0, 9999999))

docker_lock = filelock.FileLock("docker_lock")
production_setup_lock = filelock.FileLock(".exposed_ports_lock")

inline_logs = False
mt_docker_compose_file = ""
run_tenant_tests = True

try:
    requests.packages.urllib3.disable_warnings()
except:
    pass

def pytest_addoption(parser):
    parser.addoption("--clients", action="store", default="localhost:8080",
                     help="Comma-seperate mender hosts, example: 10.100.10.11:8822, 10.100.10.12:8822")
    parser.addoption("--gateway", action="store", default="127.0.0.1:8080",
                     help="Host of mender gateway")

    parser.addoption("--api", action="store", default="0.1", help="API version used in HTTP requests")
    parser.addoption("--image", action="store_true", default="core-image-full-cmdline-vexpress-qemu.ext4", help="Valid update image")
    parser.addoption("--runslow", action="store_true", help="run slow tests")
    parser.addoption("--runfast", action="store_true", help="run fast tests")
    parser.addoption("--runnightly", action="store_true", help="run nightly (very slow) tests")
    parser.addoption("--runs3", action="store_true", help="run fast tests")

    parser.addoption("--upgrade-from", action="store", help="perform upgrade test", default="")
    parser.addoption("--no-teardown", action="store_true", help="Don't tear down environment after tests are run")
    parser.addoption("--inline-logs", action="store_true", help="Don't redirect docker-compose logs to a file")

    parser.addoption("--mt-docker-compose-file", action="store", type=str, help="Docker-compose file that enables multi-tenancy (required for some tests)")

def pytest_configure(config):
    global extra_files, inline_logs, mt_docker_compose_file
    verify_sane_test_environment()

    env.api_version = config.getoption("api")
    env.valid_image = config.getoption("image")

    inline_logs = config.getoption("--inline-logs")

    if config.getoption("--mt-docker-compose-file") is None:
        if os.path.exists("../docker-compose.mt.yml"):
            logger.warn("--mt-docker-compose-file not set, but ../docker-compose.mt.yml exists, using that file.")
            mt_docker_compose_file = "-f ../docker-compose.mt.yml"
    else:
        mt_docker_compose_file = "-f %s" % config.getoption("--mt-docker-compose-file")

    env.password = ""

    # Bash not always available, nor currently required:
    env.shell = "/bin/sh -c"

    # Disable known_hosts file, to avoid "host identification changed" errors.
    env.disable_known_hosts = True

    env.abort_on_prompts = True
    # Don't allocate pseudo-TTY by default, since it is not fully functional.
    # It can still be overriden on a case by case basis by passing
    # "pty = True/False" to the various fabric functions. See
    # https://www.fabfile.org/faq.html about init scripts.
    env.always_use_pty = False

    # Don't combine stderr with stdout. The login profile sometimes prints
    # terminal specific codes there, and we don't want it interfering with our
    # output. It can still be turned on on a case by case basis by passing
    # combine_stderr to each run() or sudo() command.
    env.combine_stderr = False

    env.user = "root"

    env.connection_attempts = 50
    env.eagerly_disconnect = True
    env.banner_timeout = 10

    version = subprocess.check_output(["../extra/release_tool.py", "--version-of", "integration"])
    if re.search("(^|/)[0-9]+\.[0-9]+\.[x0-9]+", version):
        # Don't run tenant tests for release branches.
        global run_tenant_tests
        run_tenant_tests = False

def pytest_runtest_setup(item):
    logger = log.setup_custom_logger("root", item.name)
    logger.info("%s is starting.... " % item.name)

def pytest_exception_interact(node, call, report):
    if report.failed:
        for log in log_files:
            logger.info("printing content of : %s" % log)
            with open(log) as f:
                for line in f.readlines():
                    logger.info("%s: %s" % (log, line))

@pytest.mark.hookwrapper
def pytest_runtest_makereport(item, call):
    pytest_html = item.config.pluginmanager.getplugin('html')
    if pytest_html is None:
        yield
        return
    outcome = yield
    report = outcome.get_result()
    extra = getattr(report, 'extra', [])
    if report.failed:
        url = ""
        if os.getenv("UPLOAD_BACKEND_LOGS_ON_FAIL", False):
            if len(log_files) > 0:
                # we already have s3cmd configured on our build machine, so use it directly
                s3_object_name = str(uuid.uuid4()) + ".log"
                ret = subprocess.call("s3cmd put %s s3://mender-backend-logs/%s" % (log_files[-1], s3_object_name), shell=True)
                if int(ret) == 0:
                    url = "https://s3-eu-west-1.amazonaws.com/mender-backend-logs/" + s3_object_name
                else:
                    logger.warn("uploading backend logs failed.")
            else:
                logger.warn("no log files found, did the backend actually start?")
        else:
            logger.warn("not uploading backend log files because UPLOAD_BACKEND_LOGS_ON_FAIL not set")

        # always add url to report
        extra.append(pytest_html.extras.url(url))
        report.extra = extra

def pytest_unconfigure(config):
    if not config.getoption("--no-teardown"):
        stop_docker_compose()

    for log in log_files:
        try:
            os.remove(log)
        except:
            pass

def pytest_runtest_teardown(item, nextitem):
    if nextitem is None:
        stop_docker_compose()

def get_valid_image():
    return env.valid_image

def verify_sane_test_environment():
    # check if required tools are in PATH, add any other checks here
    if distutils.spawn.find_executable("mender-stress-test-client") is None:
        raise SystemExit("mender-stress-test-client not found in PATH")

    if distutils.spawn.find_executable("mender-artifact") is None:
        raise SystemExit("mender-artifact not found in PATH")

    if distutils.spawn.find_executable("docker") is None:
        raise SystemExit("docker not found in PATH")

    ret = subprocess.call("docker ps > /dev/null", shell=True)
    if ret != 0:
        raise SystemExit("not able to use docker, is your user part of the docker group?")
<|endoftext|>


<commit_before>#
# Copyright (C) 2012 - 2017 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
# pylint: disable=missing-docstring, invalid-name
from __future__ import absolute_import
import unittest

import anyconfig.backend.xml as TT
import anyconfig.backend.tests.ini
import anyconfig.tests.common
import anyconfig.compat

from anyconfig.tests.common import dicts_equal, to_bytes

XML_W_NS_S = """

   1
   <val:c>C</val:c>

"""

class Test_00(unittest.TestCase):

    def test__namespaces_from_file(self):
        ref = {"http://example.com/ns/config": '',
               "http://example.com/ns/config/val": "val"}
        xmlfile = anyconfig.compat.StringIO(XML_W_NS_S)
        self.assertTrue(dicts_equal(TT._namespaces_from_file(xmlfile), ref))

class Test_10(unittest.TestCase):

    def test_elem_to_container__None(self):
        self.assertEqual(TT.elem_to_container(None, dict, {}), dict())

    def test_root_to_container__None(self):
        self.assertEqual(TT.root_to_container(None, dict, {}), dict())

    def test_elem_to_container__attrs(self):
        ref = dict(a={"@attrs": dict(x='1', y='y')})
        root = TT.ET.XML("")
        self.assertEqual(TT.elem_to_container(root, dict, {}), ref)

    def test_elem_to_container__child(self):
        ref = dict(a=dict(b="b"))
        root = TT.ET.XML("b")
        self.assertEqual(TT.elem_to_container(root, dict, {}), ref)

    def test_elem_to_container__children(self):
        ref = {'a': {'@children': [{'b': 'b'}, {'c': 'c'}]}}
        root = TT.ET.XML("b<c>c</c>")
        self.assertEqual(TT.elem_to_container(root, dict, {}), ref)

    def test_elem_to_container__text(self):
        root = TT.ET.XML("A")
        self.assertEqual(TT.elem_to_container(root, dict, {}), {'a': 'A'})

    def test_elem_to_container__text_attrs(self):
        ref = dict(a={"@attrs": {'x': 'X'}, "@text": "A"})
        root = TT.ET.XML("A")
        self.assertEqual(TT.elem_to_container(root, dict, {}), ref)

def tree_to_string(tree):
    return TT.ET.tostring(tree.getroot())

class Test_20(unittest.TestCase):

    def test_00_container_to_etree__None(self):
        self.assertTrue(TT.container_to_etree(None) is None)

    def test_10_container_to_etree__text_attrs(self):
        ref = to_bytes('A')
        obj = dict(a={"@attrs": {'x': 'X', 'y': 'Y'}, "@text": "A"})
        res = TT.container_to_etree(obj)
        self.assertEqual(tree_to_string(res), ref)

    def test_20_container_to_etree__child(self):
        return  # TODO

        ref = to_bytes("b")
        obj = dict(a=dict(b="b"))
        res = TT.container_to_etree(obj)
        self.assertEqual(tree_to_string(res), ref)

    def test_22_container_to_etree__children(self):
        return  # TODO

        ref = to_bytes("b<c>c</c>")
        obj = {'a': {'@children': [{'b': 'b'}, {'c': 'c'}]}}
        res = TT.container_to_etree(obj)
        self.assertEqual(tree_to_string(res), ref)

# vim:sw=4:ts=4:et:
<commit_msg>fix: enable some test cases of .backend.xml.container_to_etree<commit_after>#
# Copyright (C) 2012 - 2017 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
# pylint: disable=missing-docstring, invalid-name, protected-access
from __future__ import absolute_import
import unittest

import anyconfig.backend.xml as TT
import anyconfig.backend.tests.ini
import anyconfig.tests.common
import anyconfig.compat

from anyconfig.tests.common import dicts_equal, to_bytes

XML_W_NS_S = """

   1
   <val:c>C</val:c>

"""

class Test_00(unittest.TestCase):

    def test__namespaces_from_file(self):
        ref = {"http://example.com/ns/config": '',
               "http://example.com/ns/config/val": "val"}
        xmlfile = anyconfig.compat.StringIO(XML_W_NS_S)
        self.assertTrue(dicts_equal(TT._namespaces_from_file(xmlfile), ref))

class Test_10(unittest.TestCase):

    def test_elem_to_container__None(self):
        self.assertEqual(TT.elem_to_container(None, dict, {}), dict())

    def test_root_to_container__None(self):
        self.assertEqual(TT.root_to_container(None, dict, {}), dict())

    def test_elem_to_container__attrs(self):
        ref = dict(a={"@attrs": dict(x='1', y='y')})
        root = TT.ET.XML("")
        self.assertEqual(TT.elem_to_container(root, dict, {}), ref)

    def test_elem_to_container__child(self):
        ref = dict(a=dict(b="b"))
        root = TT.ET.XML("b")
        self.assertEqual(TT.elem_to_container(root, dict, {}), ref)

    def test_elem_to_container__children(self):
        ref = {'a': {'@children': [{'b': 'b'}, {'c': 'c'}]}}
        root = TT.ET.XML("b<c>c</c>")
        self.assertEqual(TT.elem_to_container(root, dict, {}), ref)

    def test_elem_to_container__text(self):
        root = TT.ET.XML("A")
        self.assertEqual(TT.elem_to_container(root, dict, {}), {'a': 'A'})

    def test_elem_to_container__text_attrs(self):
        ref = dict(a={"@attrs": {'x': 'X'}, "@text": "A"})
        root = TT.ET.XML("A")
        self.assertEqual(TT.elem_to_container(root, dict, {}), ref)

def tree_to_string(tree):
    return TT.ET.tostring(tree.getroot())

class Test_20(unittest.TestCase):

    def test_00_container_to_etree__None(self):
        self.assertTrue(TT.container_to_etree(None) is None)

    def test_10_container_to_etree__text_attrs(self):
        ref = to_bytes('A')
        obj = dict(a={"@attrs": {'x': 'X', 'y': 'Y'}, "@text": "A"})
        res = TT.container_to_etree(obj)
        self.assertEqual(tree_to_string(res), ref)

    def test_20_container_to_etree__child(self):
        ref = to_bytes("b")
        obj = dict(a=dict(b="b"))
        res = TT.container_to_etree(obj)
        self.assertEqual(tree_to_string(res), ref)

    def test_22_container_to_etree__children(self):
        ref = to_bytes("b<c>c</c>")
        obj = {'a': {'@children': [{'b': 'b'}, {'c': 'c'}]}}
        res = TT.container_to_etree(obj)
        self.assertEqual(tree_to_string(res), ref)

# vim:sw=4:ts=4:et:
<|endoftext|>


<commit_before># gozerlib/config.py
#
#

""" config module. config is stored as item = JSON pairs. """

## gozerlib imports

from utils.trace import whichmodule
from utils.lazydict import LazyDict
from utils.exception import handle_exception
from datadir import datadir
from errors import CantSaveConfig

## simplejson imports

from simplejson import loads, dumps

## basic imports

import os
import types
import thread
import logging
import uuid

## classes

class Config(LazyDict):

    """
        config class is a dict containing json strings. is writable to file
        and human editable.

        :param filename: filename of the config file
        :type filename: string
        :param verbose: whether loading of config should ne verbose
        :type verbose: boolean
        :rtype: self

    """

    def __init__(self, filename=None, verbose=False, *args, **kw):
        LazyDict.__init__(self, *args, **kw)
        #self.dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__ + os.sep + '..')))
        self.filename = filename or 'mainconfig'
        self.dir = datadir + os.sep + 'config'
        self.cfile = self.dir + os.sep + self.filename
        self.jsondb = None

        try:
            try:
                self.fromfile(self.cfile)
            except IOError:
                pass
            import google
            from persist import Persist
            self.jsondb = Persist(self.cfile)
            self.update(self.jsondb.data)
            self.isdb = True
            logging.debug("config - fromdb - %s - %s" % (self.cfile, str(self)))
        except ImportError:
            self.isdb = False

        if not self.uuid:
            self.uuid = str(uuid.uuid4())
            self.save()

        self.init()

    def __getitem__(self, item):
        """ accessor function. """
        if not self.has_key(item):
            return None
        else:
            return dict.__getitem__(self, item)

    def set(self, item, value):
        """
            set item to value.

            :param item: item to set value of
            :type item: string
            :param value: value to set
            :type value: dict, list, string, number or boolean

        """
        dict.__setitem__(self, item, value)
        return self

    def fromdb(self):
        """ read config from database. """
        from gozerlib.persist import Persist
        logging.info("config - fromdb - %s" % self.cfile)
        tmp = Persist(self.cfile)
        self.update(tmp.data)
        return self

    def todb(self):
        """ save config to database. """
        cp = dict(self)
        del cp['jsondb']
        if not self.jsondb:
            from gozerlib.persist import Persist
            self.jsondb = Persist(self.cfile)
        self.jsondb.data = cp
        self.jsondb.save()
        return self

    def fromfile(self, filename):
        """
            read config object from filename.

            :param filename: name of file to write to
            :type filename: string
            :rtype: self

        """
        curline = ""
        # read file and set config values to loaded JSON entries
        fname = filename
        logging.info("config - fromfile - %s" % fname)
        if not os.path.exists(fname):
            return self

        # open file
        f = open(fname, 'r')

        # loop over data in config file
        for line in f:
            curline = line
            line = line.strip()

            if not line or line.startswith('#'):
                continue
            else:
                key, value = line.split('=', 1)
                self[key.strip()] = loads(unicode(value.strip()))

        return self

    def tofile(self, filename=None):
        """ save config object to file. """
        if not filename:
            filename = self.cfile

        try:
            from os import mkdir
        except ImportError:
            logging.debug("can't save %s to file .. os.mkdir() not suported" % filename)
            return

        ddir = "."
        d = []
        for p in filename.split(os.sep)[:-1]:
            d.append(p)
            ddir = os.sep.join(d)
            if not os.path.isdir(ddir):
                logging.warn("persist - creating %s dir" % ddir)
                try:
                    os.mkdir(ddir)
                except OSError, ex:
                    logging.warn("persist - not saving - failed to make %s - %s" % (ddir, str(ex)))
                    return

        written = []
        curitem = None
        try:
            # read existing config file if available
            try:
                configlist = open(filename, 'r').readlines()
            except IOError:
                configlist = []

            # make temp file
            configtmp = open(filename + '.tmp', 'w')
            teller = 0

            # write header if not already there
            if not configlist:
                configtmp.write('# %s\n\n' % self.cfile)

            # loop over original lines replacing updated data
            for line in configlist:
                teller += 1

                # skip comment
                if line.startswith('#'):
                    configtmp.write(line)
                    continue

                # take part after the =
                try:
                    keyword = line.split('=')[0].strip()
                    curitem = keyword
                except IndexError:
                    configtmp.write(line)
                    continue

                # write JSON string of data
                if self.has_key(keyword):
                    configtmp.write('%s = %s\n' % (keyword, dumps(self[keyword])))
                    written.append(keyword)
                else:
                    configtmp.write(line)

            # write data not found in original config file
            for keyword, value in self.iteritems():
                if keyword in written:
                    continue
                if keyword == 'jsondb':
                    continue
                if keyword == 'isdb':
                    continue
                if keyword == 'optionslist':
                    continue
                curitem = keyword
                configtmp.write('%s = %s\n' % (keyword, dumps(value)))

            # move temp file to original
            configtmp.close()
            try:
                os.rename(filename + '.tmp', self.cfile)
            except WindowsError:
                # no atomic operation supported on windows! error is thrown when destination exists
                os.remove(filename)
                os.rename(filename + '.tmp', self.cfile)

            return teller

        except Exception, ex:
            print "ERROR WRITING %s CONFIG FILE: %s .. %s" % (self.cfile, str(ex), curitem)

        return self

    def save(self):
        """ save the config. """
        if self.isdb:
            self.todb()
        else:
            self.tofile()

    def load(self, verbose=False):
        """
            load the config file.

            :param verbose: whether loading should be verbose

        """
        self.init()
        if self.isdb:
            self.fromdb()
        if verbose:
            logging.debug('PRE LOAD config %s' % str(self))

        return self

    def init(self):
        """ initialize the config object. """
        if self.filename == 'mainconfig':
            self.setdefault('owner', [])
            self.setdefault('loglist',  [])
            self.setdefault('quitmsg', "http://jsonbot.googlecode.com")
            self.setdefault('debug', 0)
            self.setdefault('plugdeny', [])
            self.setdefault('dotchars',  " .. ")
            self.setdefault('floodallow', 1)
            self.setdefault('auto_register', 0)
            self.setdefault('ondemand', 1)

        self['version'] = "JSONBOT 0.2 BETA2"

        return self

    def reload(self):
        """ reload the config file. """
        self.load()
        return self

def ownercheck(userhost):
    """
        check whether userhost is a owner.

        :param userhost: userhost to check
        :type userhost: string
        :rtype: boolean

    """
    if not userhost:
        return False
    if userhost in cfg['owner']:
        return True

    return False

## define

cfg = Config()
<commit_msg>check for waveapi if we need to save to database<commit_after># gozerlib/config.py
#
#

""" config module. config is stored as item = JSON pairs. """

## gozerlib imports

from utils.trace import whichmodule
from utils.lazydict import LazyDict
from utils.exception import handle_exception
from datadir import datadir
from errors import CantSaveConfig

## simplejson imports

from simplejson import loads, dumps

## basic imports

import os
import types
import thread
import logging
import uuid

## classes

class Config(LazyDict):

    """
        config class is a dict containing json strings. is writable to file
        and human editable.

        :param filename: filename of the config file
        :type filename: string
        :param verbose: whether loading of config should ne verbose
        :type verbose: boolean
        :rtype: self

    """

    def __init__(self, filename=None, verbose=False, *args, **kw):
        LazyDict.__init__(self, *args, **kw)
        #self.dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__ + os.sep + '..')))
        self.filename = filename or 'mainconfig'
        self.dir = datadir + os.sep + 'config'
        self.cfile = self.dir + os.sep + self.filename
        self.jsondb = None

        try:
            try:
                self.fromfile(self.cfile)
            except IOError:
                pass
            import waveapi
            from persist import Persist
            self.jsondb = Persist(self.cfile)
            self.update(self.jsondb.data)
            self.isdb = True
            logging.debug("config - fromdb - %s - %s" % (self.cfile, str(self)))
        except ImportError:
            self.isdb = False

        self.init()

        if not self.uuid:
            self.uuid = str(uuid.uuid4())
            self.save()

    def __getitem__(self, item):
        """ accessor function. """
        if not self.has_key(item):
            return None
        else:
            return dict.__getitem__(self, item)

    def set(self, item, value):
        """
            set item to value.

            :param item: item to set value of
            :type item: string
            :param value: value to set
            :type value: dict, list, string, number or boolean

        """
        dict.__setitem__(self, item, value)
        return self

    def fromdb(self):
        """ read config from database. """
        from gozerlib.persist import Persist
        logging.info("config - fromdb - %s" % self.cfile)
        tmp = Persist(self.cfile)
        self.update(tmp.data)
        return self

    def todb(self):
        """ save config to database. """
        cp = dict(self)
        del cp['jsondb']
        if not self.jsondb:
            from gozerlib.persist import Persist
            self.jsondb = Persist(self.cfile)
        self.jsondb.data = cp
        self.jsondb.save()
        return self

    def fromfile(self, filename):
        """
            read config object from filename.

            :param filename: name of file to write to
            :type filename: string
            :rtype: self

        """
        curline = ""
        # read file and set config values to loaded JSON entries
        fname = filename
        logging.info("config - fromfile - %s" % fname)
        if not os.path.exists(fname):
            return self

        # open file
        f = open(fname, 'r')

        # loop over data in config file
        for line in f:
            curline = line
            line = line.strip()

            if not line or line.startswith('#'):
                continue
            else:
                try:
                    key, value = line.split('=', 1)
                    self[key.strip()] = loads(unicode(value.strip()))
                except ValueError:
                    logging.warn("config - skipping line - unable to parse: %s" % line)
        return self

    def tofile(self, filename=None):
        """ save config object to file. """
        if not filename:
            filename = self.cfile

        try:
            from os import mkdir
        except ImportError:
            logging.debug("can't save %s to file .. os.mkdir() not suported" % filename)
            return

        ddir = "."
        d = []
        for p in filename.split(os.sep)[:-1]:
            d.append(p)
            ddir = os.sep.join(d)
            if not os.path.isdir(ddir):
                logging.warn("persist - creating %s dir" % ddir)
                try:
                    os.mkdir(ddir)
                except OSError, ex:
                    logging.warn("persist - not saving - failed to make %s - %s" % (ddir, str(ex)))
                    return

        written = []
        curitem = None
        try:
            # read existing config file if available
            try:
                configlist = open(filename, 'r').readlines()
            except IOError:
                configlist = []

            # make temp file
            configtmp = open(filename + '.tmp', 'w')
            teller = 0

            # write header if not already there
            if not configlist:
                configtmp.write('# %s\n\n' % self.cfile)

            # loop over original lines replacing updated data
            for line in configlist:
                teller += 1

                # skip comment
                if line.startswith('#'):
                    configtmp.write(line)
                    continue

                # take part after the =
                try:
                    keyword = line.split('=')[0].strip()
                    curitem = keyword
                except IndexError:
                    configtmp.write(line)
                    continue

                # write JSON string of data
                if self.has_key(keyword):
                    configtmp.write('%s = %s\n' % (keyword, dumps(self[keyword])))
                    written.append(keyword)
                else:
                    configtmp.write(line)

            # write data not found in original config file
            for keyword, value in self.iteritems():
                if keyword in written:
                    continue
                if keyword == 'jsondb':
                    continue
                if keyword == 'isdb':
                    continue
                if keyword == 'optionslist':
                    continue
                curitem = keyword
                configtmp.write('%s = %s\n' % (keyword, dumps(value)))

            # move temp file to original
            configtmp.close()
            try:
                os.rename(filename + '.tmp', self.cfile)
            except WindowsError:
                # no atomic operation supported on windows! error is thrown when destination exists
                os.remove(filename)
                os.rename(filename + '.tmp', self.cfile)

            return teller

        except Exception, ex:
            print "ERROR WRITING %s CONFIG FILE: %s .. %s" % (self.cfile, str(ex), curitem)

        return self

    def save(self):
        """ save the config. """
        if self.isdb:
            self.todb()
        else:
            self.tofile()

    def load(self, verbose=False):
        """
            load the config file.

            :param verbose: whether loading should be verbose

        """
        self.init()
        if self.isdb:
            self.fromdb()
        if verbose:
            logging.debug('PRE LOAD config %s' % str(self))

        return self

    def init(self):
        """ initialize the config object. """
        if self.filename == 'mainconfig':
            self.setdefault('owner', [])
            self.setdefault('loglist',  [])
            self.setdefault('quitmsg', "http://jsonbot.googlecode.com")
            self.setdefault('debug', 0)
            self.setdefault('plugdeny', [])
            self.setdefault('dotchars',  " .. ")
            self.setdefault('floodallow', 1)
            self.setdefault('auto_register', 0)
            self.setdefault('ondemand', 1)

        self['version'] = "JSONBOT 0.2 BETA2"

        return self

    def reload(self):
        """ reload the config file. """
        self.load()
        return self

def ownercheck(userhost):
    """
        check whether userhost is a owner.

        :param userhost: userhost to check
        :type userhost: string
        :rtype: boolean

    """
    if not userhost:
        return False
    if userhost in cfg['owner']:
        return True

    return False

## define

cfg = Config()
<|endoftext|>


<commit_before># Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt

"""
Pytest auto configuration.

This module is run automatically by pytest, to define and enable fixtures.
"""

import os
import sys
import warnings

import pytest

from coverage import env
from coverage.exceptions import StopEverything

# Pytest will rewrite assertions in test modules, but not elsewhere.
# This tells pytest to also rewrite assertions in coveragetest.py.
pytest.register_assert_rewrite("tests.coveragetest")
pytest.register_assert_rewrite("tests.helpers")

# Pytest can take additional options:
# $set_env.py: PYTEST_ADDOPTS - Extra arguments to pytest.

@pytest.fixture(autouse=True)
def set_warnings():
    """Configure warnings to show while running tests."""
    warnings.simplefilter("default")
    warnings.simplefilter("once", DeprecationWarning)

    # Warnings to suppress:
    # How come these warnings are successfully suppressed here, but not in setup.cfg??

    # <frozen importlib._bootstrap>:681:
    # ImportWarning: VendorImporter.exec_module() not found; falling back to load_module()
    warnings.filterwarnings(
        "ignore",
        category=ImportWarning,
        message=r".*exec_module\(\) not found; falling back to load_module\(\)",
        )
    # <frozen importlib._bootstrap>:908:
    # ImportWarning: AssertionRewritingHook.find_spec() not found; falling back to find_module()
    # <frozen importlib._bootstrap>:908:
    # ImportWarning: _SixMetaPathImporter.find_spec() not found; falling back to find_module()
    # <frozen importlib._bootstrap>:908:
    # ImportWarning: VendorImporter.find_spec() not found; falling back to find_module()
    warnings.filterwarnings(
        "ignore",
        category=ImportWarning,
        message=r".*find_spec\(\) not found; falling back to find_module\(\)",
        )

    if env.PYPY:
        # pypy3 warns about unclosed files a lot.
        warnings.filterwarnings("ignore", r".*unclosed file", category=ResourceWarning)

@pytest.fixture(autouse=True)
def reset_sys_path():
    """Clean up sys.path changes around every test."""
    sys_path = list(sys.path)
    yield
    sys.path[:] = sys_path

@pytest.fixture(autouse=True)
def fix_xdist_sys_path():
    """Prevent xdist from polluting the Python path.

    We run tests that care a lot about the contents of sys.path.  Pytest-xdist
    changes sys.path, so running with xdist, vs without xdist, sets sys.path
    differently.  With xdist, sys.path[1] is an empty string, without xdist,
    it's the virtualenv bin directory.  We don't want the empty string, so
    clobber that entry.

    See: https://github.com/pytest-dev/pytest-xdist/issues/376

    """
    if os.environ.get('PYTEST_XDIST_WORKER', ''):       # pragma: part covered
        # We are running in an xdist worker.
        if sys.path[1] == '':
            # xdist has set sys.path[1] to ''.  Clobber it.
            del sys.path[1]
        # Also, don't let it sneak stuff in via PYTHONPATH.
        try:
            del os.environ['PYTHONPATH']
        except KeyError:
            pass

@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_call(item):
    """Convert StopEverything into skipped tests."""
    outcome = yield
    if outcome.excinfo and issubclass(outcome.excinfo[0], StopEverything):
        pytest.skip(f"Skipping {item.nodeid} for StopEverything: {outcome.excinfo[1]}")
<commit_msg>test: this xdist defensiveness now seems unneeded<commit_after># Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt

"""
Pytest auto configuration.

This module is run automatically by pytest, to define and enable fixtures.
"""

import sys
import warnings

import pytest

from coverage import env
from coverage.exceptions import StopEverything

# Pytest will rewrite assertions in test modules, but not elsewhere.
# This tells pytest to also rewrite assertions in coveragetest.py.
pytest.register_assert_rewrite("tests.coveragetest")
pytest.register_assert_rewrite("tests.helpers")

# Pytest can take additional options:
# $set_env.py: PYTEST_ADDOPTS - Extra arguments to pytest.

@pytest.fixture(autouse=True)
def set_warnings():
    """Configure warnings to show while running tests."""
    warnings.simplefilter("default")
    warnings.simplefilter("once", DeprecationWarning)

    # Warnings to suppress:
    # How come these warnings are successfully suppressed here, but not in setup.cfg??

    # <frozen importlib._bootstrap>:681:
    # ImportWarning: VendorImporter.exec_module() not found; falling back to load_module()
    warnings.filterwarnings(
        "ignore",
        category=ImportWarning,
        message=r".*exec_module\(\) not found; falling back to load_module\(\)",
        )
    # <frozen importlib._bootstrap>:908:
    # ImportWarning: AssertionRewritingHook.find_spec() not found; falling back to find_module()
    # <frozen importlib._bootstrap>:908:
    # ImportWarning: _SixMetaPathImporter.find_spec() not found; falling back to find_module()
    # <frozen importlib._bootstrap>:908:
    # ImportWarning: VendorImporter.find_spec() not found; falling back to find_module()
    warnings.filterwarnings(
        "ignore",
        category=ImportWarning,
        message=r".*find_spec\(\) not found; falling back to find_module\(\)",
        )

    if env.PYPY:
        # pypy3 warns about unclosed files a lot.
        warnings.filterwarnings("ignore", r".*unclosed file", category=ResourceWarning)

@pytest.fixture(autouse=True)
def reset_sys_path():
    """Clean up sys.path changes around every test."""
    sys_path = list(sys.path)
    yield
    sys.path[:] = sys_path

@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_call(item):
    """Convert StopEverything into skipped tests."""
    outcome = yield
    if outcome.excinfo and issubclass(outcome.excinfo[0], StopEverything):
        pytest.skip(f"Skipping {item.nodeid} for StopEverything: {outcome.excinfo[1]}")
<|endoftext|>


<commit_before>from __future__ import print_function
from future import standard_library
standard_library.install_aliases()
from builtins import range
from builtins import str
from builtins import object
from past.builtins import basestring
import pytest
import os
try:
    import subprocess32 as subprocess
except:
    import subprocess
import sys
import psutil
import time
import re
import json
import urllib.request, urllib.error, urllib.parse

sys.path.append(os.getcwd())

from mrq.job import Job, queue_raw_jobs, queue_jobs
from mrq.queue import Queue
from mrq.config import get_config
from mrq.utils import wait_for_net_service
from mrq.context import connections, set_current_config

set_current_config(get_config(sources=("env")))

os.system("rm -rf dump.rdb")

class ProcessFixture(object):

    def __init__(self, request, cmdline=None, wait_port=None, quiet=False):
        self.request = request
        self.cmdline = cmdline
        self.process = None
        self.wait_port = wait_port
        self.quiet = quiet
        self.stopped = False

        self.request.addfinalizer(self.stop)

    def start(self, cmdline=None, env=None, expected_children=0):

        self.stopped = False
        self.process_children = []

        if not cmdline:
            cmdline = self.cmdline
        if env is None:
            env = {}

        # Kept from parent env
        for env_key in ["PATH", "GEVENT_LOOP", "VIRTUAL_ENV"]:
            if os.environ.get(env_key) and not env.get(env_key):
                env[env_key] = os.environ.get(env_key)

        if self.quiet:
            stdout = open(os.devnull, 'w')
        else:
            stdout = None

        self.cmdline = cmdline
        # print cmdline
        self.process = subprocess.Popen(re.split(r"\s+", cmdline) if isinstance(cmdline, basestring) else cmdline,
                                        shell=False, close_fds=True, env=env, cwd=os.getcwd(), stdout=stdout)

        if self.quiet:
            stdout.close()

        # Wait for children to start
        if expected_children > 0:
            psutil_process = psutil.Process(self.process.pid)

            # print "Expecting %s children, got %s" % (expected_children,
            # psutil_process.get_children(recursive=False))
            while True:
                self.process_children = psutil_process.get_children(
                    recursive=True)
                if len(self.process_children) >= expected_children:
                    break
                time.sleep(0.1)

        if self.wait_port:
            wait_for_net_service("127.0.0.1", int(self.wait_port), poll_interval=0.01)

    def stop(self, force=False, timeout=None, block=True, sig=15):

        # Call this only one time.
        if self.stopped and not force:
            return
        self.stopped = True

        if self.process is not None:

            os.kill(self.process.pid, sig)

            # When sending a sigkill to the process, we also want to kill the
            # children in case of supervisord usage
            if sig == 9 and len(self.process_children) > 0:
                for c in self.process_children:
                    c.send_signal(sig)

            if not block:
                return

            for _ in range(2000):

                try:
                    p = psutil.Process(self.process.pid)
                    if p.status() == "zombie":
                        # print "process %s zombie OK" % self.cmdline
                        return
                except psutil.NoSuchProcess:
                    # print "process %s exit OK" % self.cmdline
                    return

                time.sleep(0.01)

            assert False, "Process '%s' was still in state %s after 20 seconds..." % (
                self.cmdline, p.status())

class WorkerFixture(ProcessFixture):

    def __init__(self, request, **kwargs):
        ProcessFixture.__init__(self, request, cmdline=kwargs.get("cmdline"))

        self.fixture_mongodb = kwargs["mongodb"]
        self.fixture_redis = kwargs["redis"]

        self.started = False

    def start(self, flush=True, deps=True, trace=True, **kwargs):

        self.started = True

        if deps:
            self.start_deps(flush=flush)

        processes = 0
        m = re.search(r"--processes (\d+)", kwargs.get("flags", ""))
        if m:
            processes = int(m.group(1))

        cmdline = "python mrq/bin/mrq_worker.py --mongodb_logs_size 0 %s %s %s %s" % (
            "--admin_port 20020" if (processes <= 1) else "",
            "--trace_io --trace_greenlets" if trace else "",
            kwargs.get("flags", ""),
            kwargs.get("queues", "high default low")
        )

        # +1 because of supervisord itself
        if processes > 0:
            processes += 1

        print(cmdline)
        ProcessFixture.start(self, cmdline=cmdline, env=kwargs.get("env"), expected_children=processes)

    def start_deps(self, flush=True):

        self.fixture_mongodb.start()
        self.fixture_redis.start()

        # Will auto-connect
        connections.reset()

        self.mongodb_jobs = connections.mongodb_jobs
        self.mongodb_logs = connections.mongodb_logs
        self.redis = connections.redis

        if flush:
            self.fixture_mongodb.flush()
            self.fixture_redis.flush()

    def stop(self, deps=True, sig=2, **kwargs):

        if self.started:
            ProcessFixture.stop(self, sig=sig, **kwargs)

        if deps:
            self.stop_deps(**kwargs)

    def stop_deps(self, **kwargs):
        self.fixture_mongodb.stop(sig=2, **kwargs)
        self.fixture_redis.stop(sig=2, **kwargs)

    def wait_for_tasks_results(self, job_ids, block=True, accept_statuses=["success"]):

        if not block:
            return job_ids

        self.get_wait_for_idle()

        results = []

        for job_id in job_ids:
            job = Job(job_id).wait(poll_interval=0.01)
            assert job.get("status") in accept_statuses, "Job had status %s, not in %s. Dump: %s" % (
                job.get("status"), accept_statuses, job)

            results.append(job.get("result"))

        return results

    def send_raw_tasks(self, queue, params_list, start=True, block=True):
        if not self.started and start:
            self.start()

        queue_raw_jobs(queue, params_list)

        if block:
            self.get_wait_for_idle()

    def send_tasks(self, path, params_list, block=True, queue=None, accept_statuses=["success"], start=True):
        if not self.started and start:
            self.start()

        job_ids = queue_jobs(path, params_list, queue=queue)

        return self.wait_for_tasks_results(job_ids, block=block, accept_statuses=accept_statuses)

    def send_task(self, path, params, **kwargs):
        return self.send_tasks(path, [params], **kwargs)[0]

    def send_task_cli(self, path, params, queue=None, **kwargs):

        cli = ["python", "mrq/bin/mrq_run.py", "--quiet"]
        if queue:
            cli += ["--queue", queue]
        cli += [path, json.dumps(params)]

        out = subprocess.check_output(cli).strip()
        if not queue:
            return json.loads(out.decode('utf-8'))
        return out

    def get_report(self, with_memory=False):
        wait_for_net_service("127.0.0.1", 20020, poll_interval=0.01)
        f = urllib.request.urlopen("http://127.0.0.1:20020/report%s" % ("_mem" if with_memory else ""))
        data = json.loads(f.read().decode('utf-8'))
        f.close()
        return data

    def get_wait_for_idle(self):
        try:
            wait_for_net_service("127.0.0.1", 20020, poll_interval=0.01)
            f = urllib.request.urlopen("http://127.0.0.1:20020/wait_for_idle")
            data = f.read().decode('utf-8')
            assert data == "idle"
            return True
        except Exception as e:
            print("Couldn't get_wait_for_idle: %s" % e)
            return False

class RedisFixture(ProcessFixture):

    def flush(self):
        connections.redis.flushall()

        # Empty local known_queues cache too
        Queue.known_queues = {}

class MongoFixture(ProcessFixture):

    def flush(self):
        for mongodb in (connections.mongodb_jobs, connections.mongodb_logs):
            if mongodb:
                for c in mongodb.collection_names():
                    if not c.startswith("system."):
                        mongodb.drop_collection(c)

@pytest.fixture(scope="function")
def httpstatic(request):
    return ProcessFixture(request, "/usr/sbin/nginx -c /app/tests/fixtures/httpstatic/nginx.conf", wait_port=8081)

@pytest.fixture(scope="function")
def mongodb(request):
    cmd = "mongod --smallfiles --noprealloc --nojournal"
    if os.environ.get("STACK_STARTED"):
        cmd = "sleep 1h"
    return MongoFixture(request, cmd, wait_port=27017, quiet=True)

@pytest.fixture(scope="function")
def mongodb_with_journal(request):
    cmd = "mongod --smallfiles --noprealloc"
    if os.environ.get("STACK_STARTED"):
        cmd = "sleep 1h"
    return MongoFixture(request, cmd, wait_port=27017, quiet=True)

@pytest.fixture(scope="function")
def redis(request):
    cmd = "redis-server"
    if os.environ.get("STACK_STARTED"):
        cmd = "sleep 1h"
    return RedisFixture(request, cmd, wait_port=6379, quiet=True)

@pytest.fixture(scope="function")
def worker(request, mongodb, redis):

    return WorkerFixture(request, mongodb=mongodb, redis=redis)

@pytest.fixture(scope="function")
def worker_mongodb_with_journal(request, mongodb_with_journal, redis):

    return WorkerFixture(request, mongodb=mongodb_with_journal, redis=redis)
<commit_msg>another fix for psutil api change<commit_after>from __future__ import print_function
from future import standard_library
standard_library.install_aliases()
from builtins import range
from builtins import str
from builtins import object
from past.builtins import basestring
import pytest
import os
try:
    import subprocess32 as subprocess
except:
    import subprocess
import sys
import psutil
import time
import re
import json
import urllib.request, urllib.error, urllib.parse

sys.path.append(os.getcwd())

from mrq.job import Job, queue_raw_jobs, queue_jobs
from mrq.queue import Queue
from mrq.config import get_config
from mrq.utils import wait_for_net_service
from mrq.context import connections, set_current_config

set_current_config(get_config(sources=("env")))

os.system("rm -rf dump.rdb")

class ProcessFixture(object):

    def __init__(self, request, cmdline=None, wait_port=None, quiet=False):
        self.request = request
        self.cmdline = cmdline
        self.process = None
        self.wait_port = wait_port
        self.quiet = quiet
        self.stopped = False

        self.request.addfinalizer(self.stop)

    def start(self, cmdline=None, env=None, expected_children=0):

        self.stopped = False
        self.process_children = []

        if not cmdline:
            cmdline = self.cmdline
        if env is None:
            env = {}

        # Kept from parent env
        for env_key in ["PATH", "GEVENT_LOOP", "VIRTUAL_ENV"]:
            if os.environ.get(env_key) and not env.get(env_key):
                env[env_key] = os.environ.get(env_key)

        if self.quiet:
            stdout = open(os.devnull, 'w')
        else:
            stdout = None

        self.cmdline = cmdline
        # print cmdline
        self.process = subprocess.Popen(re.split(r"\s+", cmdline) if isinstance(cmdline, basestring) else cmdline,
                                        shell=False, close_fds=True, env=env, cwd=os.getcwd(), stdout=stdout)

        if self.quiet:
            stdout.close()

        # Wait for children to start
        if expected_children > 0:
            psutil_process = psutil.Process(self.process.pid)

            # print "Expecting %s children, got %s" % (expected_children,
            # psutil_process.get_children(recursive=False))
            while True:
                self.process_children = psutil_process.children(
                    recursive=True)
                if len(self.process_children) >= expected_children:
                    break
                time.sleep(0.1)

        if self.wait_port:
            wait_for_net_service("127.0.0.1", int(self.wait_port), poll_interval=0.01)

    def stop(self, force=False, timeout=None, block=True, sig=15):

        # Call this only one time.
        if self.stopped and not force:
            return
        self.stopped = True

        if self.process is not None:

            os.kill(self.process.pid, sig)

            # When sending a sigkill to the process, we also want to kill the
            # children in case of supervisord usage
            if sig == 9 and len(self.process_children) > 0:
                for c in self.process_children:
                    c.send_signal(sig)

            if not block:
                return

            for _ in range(2000):

                try:
                    p = psutil.Process(self.process.pid)
                    if p.status() == "zombie":
                        # print "process %s zombie OK" % self.cmdline
                        return
                except psutil.NoSuchProcess:
                    # print "process %s exit OK" % self.cmdline
                    return

                time.sleep(0.01)

            assert False, "Process '%s' was still in state %s after 20 seconds..." % (
                self.cmdline, p.status())

class WorkerFixture(ProcessFixture):

    def __init__(self, request, **kwargs):
        ProcessFixture.__init__(self, request, cmdline=kwargs.get("cmdline"))

        self.fixture_mongodb = kwargs["mongodb"]
        self.fixture_redis = kwargs["redis"]

        self.started = False

    def start(self, flush=True, deps=True, trace=True, **kwargs):

        self.started = True

        if deps:
            self.start_deps(flush=flush)

        processes = 0
        m = re.search(r"--processes (\d+)", kwargs.get("flags", ""))
        if m:
            processes = int(m.group(1))

        cmdline = "python mrq/bin/mrq_worker.py --mongodb_logs_size 0 %s %s %s %s" % (
            "--admin_port 20020" if (processes <= 1) else "",
            "--trace_io --trace_greenlets" if trace else "",
            kwargs.get("flags", ""),
            kwargs.get("queues", "high default low")
        )

        # +1 because of supervisord itself
        if processes > 0:
            processes += 1

        print(cmdline)
        ProcessFixture.start(self, cmdline=cmdline, env=kwargs.get("env"), expected_children=processes)

    def start_deps(self, flush=True):

        self.fixture_mongodb.start()
        self.fixture_redis.start()

        # Will auto-connect
        connections.reset()

        self.mongodb_jobs = connections.mongodb_jobs
        self.mongodb_logs = connections.mongodb_logs
        self.redis = connections.redis

        if flush:
            self.fixture_mongodb.flush()
            self.fixture_redis.flush()

    def stop(self, deps=True, sig=2, **kwargs):

        if self.started:
            ProcessFixture.stop(self, sig=sig, **kwargs)

        if deps:
            self.stop_deps(**kwargs)

    def stop_deps(self, **kwargs):
        self.fixture_mongodb.stop(sig=2, **kwargs)
        self.fixture_redis.stop(sig=2, **kwargs)

    def wait_for_tasks_results(self, job_ids, block=True, accept_statuses=["success"]):

        if not block:
            return job_ids

        self.get_wait_for_idle()

        results = []

        for job_id in job_ids:
            job = Job(job_id).wait(poll_interval=0.01)
            assert job.get("status") in accept_statuses, "Job had status %s, not in %s. Dump: %s" % (
                job.get("status"), accept_statuses, job)

            results.append(job.get("result"))

        return results

    def send_raw_tasks(self, queue, params_list, start=True, block=True):
        if not self.started and start:
            self.start()

        queue_raw_jobs(queue, params_list)

        if block:
            self.get_wait_for_idle()

    def send_tasks(self, path, params_list, block=True, queue=None, accept_statuses=["success"], start=True):
        if not self.started and start:
            self.start()

        job_ids = queue_jobs(path, params_list, queue=queue)

        return self.wait_for_tasks_results(job_ids, block=block, accept_statuses=accept_statuses)

    def send_task(self, path, params, **kwargs):
        return self.send_tasks(path, [params], **kwargs)[0]

    def send_task_cli(self, path, params, queue=None, **kwargs):

        cli = ["python", "mrq/bin/mrq_run.py", "--quiet"]
        if queue:
            cli += ["--queue", queue]
        cli += [path, json.dumps(params)]

        out = subprocess.check_output(cli).strip()
        if not queue:
            return json.loads(out.decode('utf-8'))
        return out

    def get_report(self, with_memory=False):
        wait_for_net_service("127.0.0.1", 20020, poll_interval=0.01)
        f = urllib.request.urlopen("http://127.0.0.1:20020/report%s" % ("_mem" if with_memory else ""))
        data = json.loads(f.read().decode('utf-8'))
        f.close()
        return data

    def get_wait_for_idle(self):
        try:
            wait_for_net_service("127.0.0.1", 20020, poll_interval=0.01)
            f = urllib.request.urlopen("http://127.0.0.1:20020/wait_for_idle")
            data = f.read().decode('utf-8')
            assert data == "idle"
            return True
        except Exception as e:
            print("Couldn't get_wait_for_idle: %s" % e)
            return False

class RedisFixture(ProcessFixture):

    def flush(self):
        connections.redis.flushall()

        # Empty local known_queues cache too
        Queue.known_queues = {}

class MongoFixture(ProcessFixture):

    def flush(self):
        for mongodb in (connections.mongodb_jobs, connections.mongodb_logs):
            if mongodb:
                for c in mongodb.collection_names():
                    if not c.startswith("system."):
                        mongodb.drop_collection(c)

@pytest.fixture(scope="function")
def httpstatic(request):
    return ProcessFixture(request, "/usr/sbin/nginx -c /app/tests/fixtures/httpstatic/nginx.conf", wait_port=8081)

@pytest.fixture(scope="function")
def mongodb(request):
    cmd = "mongod --smallfiles --noprealloc --nojournal"
    if os.environ.get("STACK_STARTED"):
        cmd = "sleep 1h"
    return MongoFixture(request, cmd, wait_port=27017, quiet=True)

@pytest.fixture(scope="function")
def mongodb_with_journal(request):
    cmd = "mongod --smallfiles --noprealloc"
    if os.environ.get("STACK_STARTED"):
        cmd = "sleep 1h"
    return MongoFixture(request, cmd, wait_port=27017, quiet=True)

@pytest.fixture(scope="function")
def redis(request):
    cmd = "redis-server"
    if os.environ.get("STACK_STARTED"):
        cmd = "sleep 1h"
    return RedisFixture(request, cmd, wait_port=6379, quiet=True)

@pytest.fixture(scope="function")
def worker(request, mongodb, redis):

    return WorkerFixture(request, mongodb=mongodb, redis=redis)

@pytest.fixture(scope="function")
def worker_mongodb_with_journal(request, mongodb_with_journal, redis):

    return WorkerFixture(request, mongodb=mongodb_with_journal, redis=redis)
<|endoftext|>


<commit_before># Copyright 2020 Richard Koshak
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains DhtSensor, a sensor that reads DHT11, DHT22 or AM2302 temp/humi
sensors. Reading of the sensor is accomplished using the adafruit_dht library.
"""
from configparser import NoOptionError
import board
import adafruit_dht
from core.sensor import Sensor

class DhtSensor(Sensor):
    """A polling sensor that reads and reports temperature and humidity. It
    supports DHT22, AM2302, and DHT11 sensors. It requires a poll > 0.
    Parameters:
        "Pin" : the data pin the sensor is connected to
        "Sensor": one of "DHT22", "DHT11", "AM2302". It must match the sensor
        type.
        "HumiDest": destination for the humidity reading
        "TempDest": destination for the temp reading
        "TempUnit": optional, one of "C" or "F", units the temp is published,
        defaults to "C"
        "Smoothing": optional parameter, when True it will publish the average
        of the last five readings instead of just the current reading.

    Raises:
        NoOptionError when a required options is not present.
        ValueError when a parameter has an unsupported value.
    """

    def __init__(self, publishers, params):
        super().__init__(publishers, params)

        if self.poll <= 0:
            raise ValueError("A positive polling period is required: {}"
                             .format(self.poll))

        pin = params("Pin")
        bpin = None
        pin_name = "D{}".format(pin)
        if hasattr(board, pin_name):
            bpin = getattr(board, pin_name)
        else:
            raise ValueError("Unsupported pin number {}".format(pin))

        sen_type = params("Sensor")
        if sen_type in ("DHT22", "AM2302"):
            self.log.info("Creating DHT22/AM2302 sensor")
            self.sensor = adafruit_dht.DHT22(bpin)
        elif sen_type == "DHT11":
            self.sensor = adafruit_dht.DHT11(bpin)
        else:
            raise ValueError("{} is an unsupported Sensor".format(sen_type))

        self.log.info("Sensor created, setting parameters.")
        self.humi_dest = params("HumiDest")
        self.temp_dest = params("TempDest")

        # Default to C. If it's defined and not C or F raises ValueError.
        try:
            self.temp_unit = params("TempUnit")
            if self.temp_unit not in ("C", "F"):
                raise ValueError("{} is an unsupported temp unit".format(self.temp_unit))
        except NoOptionError:
            self.temp_unit = "C"

        try:
            self.smoothing = bool(params("Smoothing"))
        except NoOptionError:
            self.smoothing = False

        if self.smoothing:
            self.humidity_readings = [None] * 5
            self.temp_readings = [None] * 5

        self.publish_state()

    def publish_state(self):
        """Acquires the current reading. If the value is reasonable (humidity
        between 0 and 100, temp between -40 and 125) the reading is published.
        If smoothing, the average of the most recent five readings is published.
        If not smoothing the current reading is published. If temp_unit is "F",
        the temp is published in degrees F. Both temp and humidity are rounded
        to the tenth's place.
        Warning log statements are written for unreasonable values or errors
        reading the sensor.
        """
        try:
            temp = self.sensor.temperature
            if temp and self.temp_unit == "F":
                temp = temp * (9 / 5) + 32

            humidity = self.sensor.humidity

            if temp and -40 <= temp <= 125:
                to_send = temp
                if self.smoothing:
                    self.temp_readings.pop()
                    self.temp_readings.insert(0, temp)
                    to_send = sum([t for t in self.temp_readings if t]) / 5
                self._send("{:.1f}".format(to_send), self.temp_dest)
            else:
                self.log.warning("Unreasonable temperature reading of %s "
                                 "dropping it", temp)

            if humidity and 0 <= humidity <= 100:
                to_send = humidity
                if self.smoothing:
                    self.humidity_readings.pop()
                    self.humidity_readings.insert(0, humidity)
                    to_send = sum([h for h in self.humidity_readings if h]) / 5
                self._send("{:.1f}".format(to_send), self.humi_dest)
            else:
                self.log.warning("Unreasonable humidity reading of %s, "
                                 "dropping it", humidity)

        except RuntimeError as error:
            self.log.warning("Error reading DHT: %s", error.args[0])
<commit_msg>Added docstrings<commit_after># Copyright 2020 Richard Koshak
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains DhtSensor, a sensor that reads DHT11, DHT22 or AM2302 temp/humi
sensors. Reading of the sensor is accomplished using the adafruit_dht library.
"""
from configparser import NoOptionError
import board
import adafruit_dht
from core.sensor import Sensor

class DhtSensor(Sensor):
    """A polling sensor that reads and reports temperature and humidity. It
    supports DHT22, AM2302, and DHT11 sensors. It requires a poll > 0.
    Parameters:
        "Pin" : the data pin the sensor is connected to
        "Sensor": one of "DHT22", "DHT11", "AM2302". It must match the sensor
        type.
        "HumiDest": destination for the humidity reading
        "TempDest": destination for the temp reading
        "TempUnit": optional, one of "C" or "F", units the temp is published,
        defaults to "C"
        "Smoothing": optional parameter, when True it will publish the average
        of the last five readings instead of just the current reading.

    Raises:
        NoOptionError when a required options is not present.
        ValueError when a parameter has an unsupported value.
    """

    def __init__(self, publishers, params):
        """Initialize the DHT sensor and collect the first reading.
        Parameters:
            - Poll: must be > 0
            - Pin: GPIO pin where the data pin of the sensor is wired, BMC
            numbering
            - Sensor: one of "DHT11", "DHT22" or "AM2302"
            - HumiDest: destination for the humidity readings
            - TempDest: destination for the temperature readings
            - TempUnit: optional parameter, one of "C" or "F", defaults to "C"
            - Smoothing: optional parameter, if True the average of the last
            five readings is published, when False only the most recent is
            published.
        Raises
            - NoOptionError: if a required parameter is not present
            - ValueError: if a parameter has an unsuable value
            - RuntimeError: if there is a problem connecting to the sensor:w

        """
        super().__init__(publishers, params)

        if self.poll <= 0:
            raise ValueError("A positive polling period is required: {}"
                             .format(self.poll))

        pin = params("Pin")
        bpin = None
        pin_name = "D{}".format(pin)
        if hasattr(board, pin_name):
            bpin = getattr(board, pin_name)
        else:
            raise ValueError("Unsupported pin number {}".format(pin))

        sen_type = params("Sensor")
        if sen_type in ("DHT22", "AM2302"):
            self.log.info("Creating DHT22/AM2302 sensor")
            self.sensor = adafruit_dht.DHT22(bpin)
        elif sen_type == "DHT11":
            self.sensor = adafruit_dht.DHT11(bpin)
        else:
            raise ValueError("{} is an unsupported Sensor".format(sen_type))

        self.log.info("Sensor created, setting parameters.")
        self.humi_dest = params("HumiDest")
        self.temp_dest = params("TempDest")

        # Default to C. If it's defined and not C or F raises ValueError.
        try:
            self.temp_unit = params("TempUnit")
            if self.temp_unit not in ("C", "F"):
                raise ValueError("{} is an unsupported temp unit".format(self.temp_unit))
        except NoOptionError:
            self.temp_unit = "C"

        try:
            self.smoothing = bool(params("Smoothing"))
        except NoOptionError:
            self.smoothing = False

        if self.smoothing:
            self.humidity_readings = [None] * 5
            self.temp_readings = [None] * 5

        self.publish_state()

    def publish_state(self):
        """Acquires the current reading. If the value is reasonable (humidity
        between 0 and 100, temp between -40 and 125) the reading is published.
        If smoothing, the average of the most recent five readings is published.
        If not smoothing the current reading is published. If temp_unit is "F",
        the temp is published in degrees F. Both temp and humidity are rounded
        to the tenth's place.
        Warning log statements are written for unreasonable values or errors
        reading the sensor.
        """
        try:
            temp = self.sensor.temperature
            if temp and self.temp_unit == "F":
                temp = temp * (9 / 5) + 32

            humidity = self.sensor.humidity

            if temp and -40 <= temp <= 125:
                to_send = temp
                if self.smoothing:
                    self.temp_readings.pop()
                    self.temp_readings.insert(0, temp)
                    to_send = sum([t for t in self.temp_readings if t]) / 5
                self._send("{:.1f}".format(to_send), self.temp_dest)
            else:
                self.log.warning("Unreasonable temperature reading of %s "
                                 "dropping it", temp)

            if humidity and 0 <= humidity <= 100:
                to_send = humidity
                if self.smoothing:
                    self.humidity_readings.pop()
                    self.humidity_readings.insert(0, humidity)
                    to_send = sum([h for h in self.humidity_readings if h]) / 5
                self._send("{:.1f}".format(to_send), self.humi_dest)
            else:
                self.log.warning("Unreasonable humidity reading of %s, "
                                 "dropping it", humidity)

        except RuntimeError as error:
            self.log.warning("Error reading DHT: %s", error.args[0])
<|endoftext|>


<commit_before>import os
from getpass import getuser
import pytest
from pathlib import Path
import shutil
import subprocess
import responses
import tempfile
from unittest.mock import patch, Mock

def _get_temp_dir():
    return Path(tempfile.mkdtemp())

@pytest.fixture(scope="session")
def local_pip_cache():
    try:
        return next(Path(tempfile.gettempdir()).glob('*/test-pip-cache'))
    except StopIteration:
        return _get_temp_dir() / 'test-pip-cache'

@pytest.fixture
def fake_home(local_pip_cache):
    tempdir = _get_temp_dir()
    cache_dir = tempdir / '.cache'
    cache_dir.mkdir()
    (cache_dir / 'pip').symlink_to(local_pip_cache)

    old_home = os.environ['HOME']
    old_home_contents = set(Path(old_home).iterdir())

    os.environ['HOME'] = str(tempdir)
    yield tempdir
    os.environ['HOME'] = old_home
    shutil.rmtree(tempdir)

    new_stuff = set(Path(old_home).iterdir()) - old_home_contents
    if new_stuff:
        raise Exception('home mocking failed somewehere: {}, {}'.format(
            new_stuff, tempdir
        ))

@pytest.fixture
def virtualenvs_folder():
    actual_virtualenvs = Path(f'/home/{getuser()}/.virtualenvs')
    old_virtualenvs = set(Path(actual_virtualenvs).iterdir())

    tempdir = _get_temp_dir()
    old_workon = os.environ['WORKON_HOME']
    os.environ['WORKON_HOME'] = str(tempdir)
    yield tempdir
    os.environ['WORKON_HOME'] = old_workon
    shutil.rmtree(tempdir)

    new_envs = set(actual_virtualenvs.iterdir()) - set(old_virtualenvs)
    if new_envs:
        raise Exception('virtualenvs path mocking failed somewehere: {}, {}'.format(
            new_envs, tempdir
        ))

@pytest.fixture
def test_virtualenv(virtualenvs_folder):
    virtualenv_name = Path(tempfile.NamedTemporaryFile().name).name
    subprocess.check_output([
        'bash', '-c',
        'source virtualenvwrapper.sh && mkvirtualenv {} && pip install django==1.8.7'.format(
            virtualenv_name
        )
    ])
    return virtualenvs_folder / virtualenv_name

@pytest.fixture
def mock_subprocess():
    mock = Mock()
    with patch('subprocess.check_call') as mock_check_call:
        mock.check_call = mock_check_call
        with patch('subprocess.check_output') as mock_check_output:
            mock.check_output = mock_check_output
            yield mock

@pytest.fixture
def api_responses():
    with responses.RequestsMock() as r:
        yield r

@pytest.fixture(scope="function")
def api_token():
    old_token = os.environ.get('API_TOKEN')
    token = 'sekrit.token'
    os.environ['API_TOKEN'] = token

    yield token

    if old_token is None:
        del os.environ['API_TOKEN']
    else:
        os.environ['API_TOKEN'] = old_token

@pytest.fixture(scope="function")
def no_api_token():
    if 'API_TOKEN' not in os.environ:
        yield

    else:
        old_token = os.environ['API_TOKEN']
        del os.environ['API_TOKEN']
        yield
        os.environ['API_TOKEN'] = old_token

<commit_msg>use pytest cache thing instead, thanks @tomviner<commit_after>import os
from getpass import getuser
import pytest
from pathlib import Path
import shutil
import subprocess
import responses
import tempfile
from unittest.mock import patch, Mock

def _get_temp_dir():
    return Path(tempfile.mkdtemp())

@pytest.fixture(scope="session")
def local_pip_cache(request):
    previous_cache = request.config.cache.get('pythonanywhere/pip-cache', None)
    if previous_cache:
        return Path(previous_cache)
    else:
        new_cache = _get_temp_dir()
        request.config.cache.set('pythonanywhere/pip-cache', str(new_cache))
        return new_cache

@pytest.fixture
def fake_home(local_pip_cache):
    tempdir = _get_temp_dir()
    cache_dir = tempdir / '.cache'
    cache_dir.mkdir()
    (cache_dir / 'pip').symlink_to(local_pip_cache)

    old_home = os.environ['HOME']
    old_home_contents = set(Path(old_home).iterdir())

    os.environ['HOME'] = str(tempdir)
    yield tempdir
    os.environ['HOME'] = old_home
    shutil.rmtree(tempdir)

    new_stuff = set(Path(old_home).iterdir()) - old_home_contents
    if new_stuff:
        raise Exception('home mocking failed somewehere: {}, {}'.format(
            new_stuff, tempdir
        ))

@pytest.fixture
def virtualenvs_folder():
    actual_virtualenvs = Path(f'/home/{getuser()}/.virtualenvs')
    old_virtualenvs = set(Path(actual_virtualenvs).iterdir())

    tempdir = _get_temp_dir()
    old_workon = os.environ['WORKON_HOME']
    os.environ['WORKON_HOME'] = str(tempdir)
    yield tempdir
    os.environ['WORKON_HOME'] = old_workon
    shutil.rmtree(tempdir)

    new_envs = set(actual_virtualenvs.iterdir()) - set(old_virtualenvs)
    if new_envs:
        raise Exception('virtualenvs path mocking failed somewehere: {}, {}'.format(
            new_envs, tempdir
        ))

@pytest.fixture
def test_virtualenv(virtualenvs_folder):
    virtualenv_name = Path(tempfile.NamedTemporaryFile().name).name
    subprocess.check_output([
        'bash', '-c',
        'source virtualenvwrapper.sh && mkvirtualenv {} && pip install django==1.8.7'.format(
            virtualenv_name
        )
    ])
    return virtualenvs_folder / virtualenv_name

@pytest.fixture
def mock_subprocess():
    mock = Mock()
    with patch('subprocess.check_call') as mock_check_call:
        mock.check_call = mock_check_call
        with patch('subprocess.check_output') as mock_check_output:
            mock.check_output = mock_check_output
            yield mock

@pytest.fixture
def api_responses():
    with responses.RequestsMock() as r:
        yield r

@pytest.fixture(scope="function")
def api_token():
    old_token = os.environ.get('API_TOKEN')
    token = 'sekrit.token'
    os.environ['API_TOKEN'] = token

    yield token

    if old_token is None:
        del os.environ['API_TOKEN']
    else:
        os.environ['API_TOKEN'] = old_token

@pytest.fixture(scope="function")
def no_api_token():
    if 'API_TOKEN' not in os.environ:
        yield

    else:
        old_token = os.environ['API_TOKEN']
        del os.environ['API_TOKEN']
        yield
        os.environ['API_TOKEN'] = old_token

<|endoftext|>