<commit_before>from os import path
import sys
import traceback

from cupy.cuda.memory_hook import MemoryHook

class LineProfileHook(MemoryHook):
    """Code line Cupy memory profiler.

    This profiler shows line-by-line memory consumption using traceback module.
    But, note that it can trace only cpython level, no cython level.
    ref. https://github.com/cython/cython/issues/1755

    Example:
        Code example::

            from cupy.cuda import memory_hooks
            hook = memory_hooks.LineProfileHook()
            with hook:
                # some cupy codes
            hook.print_report()

        Output example::

            _root (4.00KB, 4.00KB)
              lib/python3.6/unittest/__main__.py:18:<module> (4.00KB, 4.00KB)
                lib/python3.6/unittest/main.py:255:runTests (4.00KB, 4.00KB)
                  tests/cupy_tests/test.py:37:test (1.00KB, 1.00KB)
                  tests/cupy_tests/test.py:38:test (1.00KB, 1.00KB)
                  tests/cupy_tests/test.py:39:test (2.00KB, 2.00KB)

        Each line shows::

            {filename}:{lineno}:{func_name} ({used_bytes}, {acquired_bytes})

        where *used_bytes* is the memory bytes used from cupy memory pool, and
        *acquired_bytes* is the actual memory bytes the cupy memory pool
        acquired from GPU device.
        *_root* is a root node of the stack trace to show total memory usage.

    Args:
        max_depth (int): maximum depth to follow stack traces.
            Default is 0 (no limit).
    """

    name = 'LineProfileHook'

    def __init__(self, max_depth=0):
        self._memory_frames = {}
        self._root = MemoryFrame(None, None)
        self._filename = path.abspath(__file__)
        self._max_depth = max_depth

    # callback
    def malloc_preprocess(self, device_id, size, mem_size):
        self._cretate_frame_tree(used_bytes=mem_size)

    # callback
    def alloc_preprocess(self, device_id, mem_size):
        self._cretate_frame_tree(acquired_bytes=mem_size)

    def _cretate_frame_tree(self, used_bytes=0, acquired_bytes=0):
        self._root.used_bytes += used_bytes
        self._root.acquired_bytes += acquired_bytes
        parent = self._root
        for depth, stackframe in enumerate(self._extract_stackframes()):
            if self._max_depth > 0 and self._max_depth <= depth + 1:
                break
            memory_frame = self._add_frame(parent, stackframe)
            memory_frame.used_bytes += used_bytes
            memory_frame.acquired_bytes += acquired_bytes
            parent = memory_frame

    def _extract_stackframes(self):
        stackframes = traceback.extract_stack()
        stackframes = [StackFrame(st) for st in stackframes]
        stackframes = [
            st for st in stackframes if st.filename != self._filename]
        return stackframes

    def _key_frame(self, parent, stackframe):
        return (parent,
                stackframe.filename,
                stackframe.lineno,
                stackframe.name)

    def _add_frame(self, parent, stackframe):
        key = self._key_frame(parent, stackframe)
        if key in self._memory_frames:
            memory_frame = self._memory_frames[key]
        else:
            memory_frame = MemoryFrame(parent, stackframe)
            self._memory_frames[key] = memory_frame
        return memory_frame

    def print_report(self, file=sys.stdout):
        """Prints a report of line memory profiling."""
        line = '_root (%s, %s)\n' % self._root.humanized_bytes()
        file.write(line)
        for child in self._root.children:
            self._print_frame(child, depth=1, file=file)
        file.flush()

    def _print_frame(self, memory_frame, depth=0, file=sys.stdout):
        indent = ' ' * (depth * 2)
        st = memory_frame.stackframe
        used_bytes, acquired_bytes = memory_frame.humanized_bytes()
        line = '%s%s:%s:%s (%s, %s)\n' % (
            indent, st.filename, st.lineno, st.name,
            used_bytes, acquired_bytes)
        file.write(line)
        for child in memory_frame.children:
            self._print_frame(child, depth=depth + 1, file=file)

class StackFrame(object):
    """Compatibility layer for outputs of traceback.extract_stack()

    Attributes:
        filename (string): filename
        lineno (int): line number
        name (string): function name
    """

    def __init__(self, obj):
        if isinstance(obj, tuple):  # < 3.5
            self.filename = obj[0]
            self.lineno = obj[1]
            self.name = obj[2]
        else:  # >= 3.5 FrameSummary
            self.filename = obj.filename
            self.lineno = obj.lineno
            self.name = obj.name

class MemoryFrame(object):
    """A single stack frame along with sum of memory usage at the frame

    Attributes:
        stackframe (FrameSummary): stackframe from traceback.extract_stack().
        parent (MemoryFrame): parent frame, that is, caller.
        children (list of MemoryFrame): child frames, that is, callees.
        used_bytes (int): memory bytes that users used from cupy memory pool.
        acquired_bytes (int): memory bytes that cupy memory pool acquired
            from GPU device.
    """

    def __init__(self, parent, stackframe):
        self.stackframe = stackframe
        self.children = []
        self._set_parent(parent)
        self.used_bytes = 0
        self.acquired_bytes = 0

    def humanized_bytes(self):
        used_bytes = self._humanized_size(self.used_bytes)
        acquired_bytes = self._humanized_size(self.acquired_bytes)
        return (used_bytes, acquired_bytes)

    def _set_parent(self, parent):
        if parent and parent not in parent.children:
            self.parent = parent
            parent.children.append(self)

    def _humanized_size(self, size):
        for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E']:
            if size < 1024.0:
                return '%3.2f%sB' % (size, unit)
            size /= 1024.0
        return '%.2f%sB' % (size, 'Z')
<commit_msg>fix wording of document comments<commit_after>from os import path
import sys
import traceback

from cupy.cuda.memory_hook import MemoryHook

class LineProfileHook(MemoryHook):
    """Code line CuPy memory profiler.

    This profiler shows line-by-line GPU memory consumption using traceback
    module. But, note that it can trace only CPython level, no Cython level.
    ref. https://github.com/cython/cython/issues/1755

    Example:
        Code example::

            from cupy.cuda import memory_hooks
            hook = memory_hooks.LineProfileHook()
            with hook:
                # some CuPy codes
            hook.print_report()

        Output example::

            _root (4.00KB, 4.00KB)
              lib/python3.6/unittest/__main__.py:18:<module> (4.00KB, 4.00KB)
                lib/python3.6/unittest/main.py:255:runTests (4.00KB, 4.00KB)
                  tests/cupy_tests/test.py:37:test (1.00KB, 1.00KB)
                  tests/cupy_tests/test.py:38:test (1.00KB, 1.00KB)
                  tests/cupy_tests/test.py:39:test (2.00KB, 2.00KB)

        Each line shows::

            {filename}:{lineno}:{func_name} ({used_bytes}, {acquired_bytes})

        where *used_bytes* is the memory bytes used from CuPy memory pool, and
        *acquired_bytes* is the actual memory bytes the CuPy memory pool
        acquired from GPU device.
        *_root* is a root node of the stack trace to show total memory usage.

    Args:
        max_depth (int): maximum depth to follow stack traces.
            Default is 0 (no limit).
    """

    name = 'LineProfileHook'

    def __init__(self, max_depth=0):
        self._memory_frames = {}
        self._root = MemoryFrame(None, None)
        self._filename = path.abspath(__file__)
        self._max_depth = max_depth

    # callback
    def malloc_preprocess(self, device_id, size, mem_size):
        self._cretate_frame_tree(used_bytes=mem_size)

    # callback
    def alloc_preprocess(self, device_id, mem_size):
        self._cretate_frame_tree(acquired_bytes=mem_size)

    def _cretate_frame_tree(self, used_bytes=0, acquired_bytes=0):
        self._root.used_bytes += used_bytes
        self._root.acquired_bytes += acquired_bytes
        parent = self._root
        for depth, stackframe in enumerate(self._extract_stackframes()):
            if self._max_depth > 0 and self._max_depth <= depth + 1:
                break
            memory_frame = self._add_frame(parent, stackframe)
            memory_frame.used_bytes += used_bytes
            memory_frame.acquired_bytes += acquired_bytes
            parent = memory_frame

    def _extract_stackframes(self):
        stackframes = traceback.extract_stack()
        stackframes = [StackFrame(st) for st in stackframes]
        stackframes = [
            st for st in stackframes if st.filename != self._filename]
        return stackframes

    def _key_frame(self, parent, stackframe):
        return (parent,
                stackframe.filename,
                stackframe.lineno,
                stackframe.name)

    def _add_frame(self, parent, stackframe):
        key = self._key_frame(parent, stackframe)
        if key in self._memory_frames:
            memory_frame = self._memory_frames[key]
        else:
            memory_frame = MemoryFrame(parent, stackframe)
            self._memory_frames[key] = memory_frame
        return memory_frame

    def print_report(self, file=sys.stdout):
        """Prints a report of line memory profiling."""
        line = '_root (%s, %s)\n' % self._root.humanized_bytes()
        file.write(line)
        for child in self._root.children:
            self._print_frame(child, depth=1, file=file)
        file.flush()

    def _print_frame(self, memory_frame, depth=0, file=sys.stdout):
        indent = ' ' * (depth * 2)
        st = memory_frame.stackframe
        used_bytes, acquired_bytes = memory_frame.humanized_bytes()
        line = '%s%s:%s:%s (%s, %s)\n' % (
            indent, st.filename, st.lineno, st.name,
            used_bytes, acquired_bytes)
        file.write(line)
        for child in memory_frame.children:
            self._print_frame(child, depth=depth + 1, file=file)

class StackFrame(object):
    """Compatibility layer for outputs of traceback.extract_stack().

    Attributes:
        filename (string): filename
        lineno (int): line number
        name (string): function name
    """

    def __init__(self, obj):
        if isinstance(obj, tuple):  # < 3.5
            self.filename = obj[0]
            self.lineno = obj[1]
            self.name = obj[2]
        else:  # >= 3.5 FrameSummary
            self.filename = obj.filename
            self.lineno = obj.lineno
            self.name = obj.name

class MemoryFrame(object):
    """A single stack frame along with sum of memory usage at the frame.

    Attributes:
        stackframe (FrameSummary): stackframe from traceback.extract_stack().
        parent (MemoryFrame): parent frame, that is, caller.
        children (list of MemoryFrame): child frames, that is, callees.
        used_bytes (int): memory bytes that users used from CuPy memory pool.
        acquired_bytes (int): memory bytes that CuPy memory pool acquired
            from GPU device.
    """

    def __init__(self, parent, stackframe):
        self.stackframe = stackframe
        self.children = []
        self._set_parent(parent)
        self.used_bytes = 0
        self.acquired_bytes = 0

    def humanized_bytes(self):
        used_bytes = self._humanized_size(self.used_bytes)
        acquired_bytes = self._humanized_size(self.acquired_bytes)
        return (used_bytes, acquired_bytes)

    def _set_parent(self, parent):
        if parent and parent not in parent.children:
            self.parent = parent
            parent.children.append(self)

    def _humanized_size(self, size):
        for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E']:
            if size < 1024.0:
                return '%3.2f%sB' % (size, unit)
            size /= 1024.0
        return '%.2f%sB' % (size, 'Z')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import hashlib
import random
import string
import syutil
from syutil.base64util import encode_base64

from twisted.web.resource import Resource

import json
from sydent.db.invite_tokens import JoinTokenStore
from sydent.db.threepid_associations import GlobalAssociationStore

from sydent.http.servlets import require_args, send_cors
from sydent.util.emailutils import sendEmail

class StoreInviteServlet(Resource):
    def __init__(self, syd):
        self.sydent = syd

    def render_POST(self, request):
        send_cors(request)
        err = require_args(request, ("medium", "address", "room_id", "sender",))
        if err:
            return json.dumps(err)
        medium = request.args["medium"][0]
        address = request.args["address"][0]
        roomId = request.args["room_id"][0]
        sender = request.args["sender"][0]

        globalAssocStore = GlobalAssociationStore(self.sydent)
        mxid = globalAssocStore.getMxid(medium, address)
        if mxid:
            request.setResponseCode(400)
            return json.dumps({
                "errcode": "THREEPID_IN_USE",
                "error": "Binding already known",
                "mxid": mxid,
            })

        if medium != "email":
            request.setResponseCode(400)
            return json.dumps({
                "errcode": "M_UNRECOGNIZED",
                "error": "Didn't understand medium '%s'" % (medium,),
            })

        token = self._randomString(128)

        JoinTokenStore(self.sydent).storeToken(medium, address, roomId, sender, token)

        substitutions = {}
        for key, values in request.args:
            if len(values) == 1 and type(values[0]) == str:
                substitutions[key] = values[0]

        sendEmail(self.sydent, "email.invite_template", address, substitutions)

        pubKey = self.sydent.keyring.ed25519.verify_key
        pubKeyBase64 = encode_base64(pubKey.encode())

        resp = {
            "token": token,
            "public_key": pubKeyBase64,
            "display_name": self.redact(address),
        }

        return json.dumps(resp)

    def redact(self, address):
        return "@".join(map(self._redact, address.split("@", 1)))

    def _redact(self, s):
        if len(s) > 5:
            return s[:3] + "..."
        elif len(s) > 1:
            return s[0] + "..."
        else:
            return "..."

    def _randomString(self, length):
        return ''.join(random.choice(string.ascii_letters) for _ in xrange(length))
<commit_msg>Escape substitution bodies<commit_after># -*- coding: utf-8 -*-

# Copyright 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import cgi
import hashlib
import random
import string
import syutil
from syutil.base64util import encode_base64

from twisted.web.resource import Resource

import json
from sydent.db.invite_tokens import JoinTokenStore
from sydent.db.threepid_associations import GlobalAssociationStore

from sydent.http.servlets import require_args, send_cors
from sydent.util.emailutils import sendEmail

class StoreInviteServlet(Resource):
    def __init__(self, syd):
        self.sydent = syd

    def render_POST(self, request):
        send_cors(request)
        err = require_args(request, ("medium", "address", "room_id", "sender",))
        if err:
            return json.dumps(err)
        medium = request.args["medium"][0]
        address = request.args["address"][0]
        roomId = request.args["room_id"][0]
        sender = request.args["sender"][0]

        globalAssocStore = GlobalAssociationStore(self.sydent)
        mxid = globalAssocStore.getMxid(medium, address)
        if mxid:
            request.setResponseCode(400)
            return json.dumps({
                "errcode": "THREEPID_IN_USE",
                "error": "Binding already known",
                "mxid": mxid,
            })

        if medium != "email":
            request.setResponseCode(400)
            return json.dumps({
                "errcode": "M_UNRECOGNIZED",
                "error": "Didn't understand medium '%s'" % (medium,),
            })

        token = self._randomString(128)

        JoinTokenStore(self.sydent).storeToken(medium, address, roomId, sender, token)

        substitutions = {}
        for key, values in request.args:
            if len(values) == 1 and type(values[0]) == str:
                substitutions[key] = cgi.escape(values[0])

        sendEmail(self.sydent, "email.invite_template", address, substitutions)

        pubKey = self.sydent.keyring.ed25519.verify_key
        pubKeyBase64 = encode_base64(pubKey.encode())

        resp = {
            "token": token,
            "public_key": pubKeyBase64,
            "display_name": self.redact(address),
        }

        return json.dumps(resp)

    def redact(self, address):
        return "@".join(map(self._redact, address.split("@", 1)))

    def _redact(self, s):
        if len(s) > 5:
            return s[:3] + "..."
        elif len(s) > 1:
            return s[0] + "..."
        else:
            return "..."

    def _randomString(self, length):
        return ''.join(random.choice(string.ascii_letters) for _ in xrange(length))
<|endoftext|>


<commit_before>{
  'targets': [
    {'target_name': 'libluvit',
      'type': 'static_library',
      'dependencies': [
        'deps/http-parser/http_parser.gyp:http_parser',
        'deps/luajit.gyp:luajit',
        'deps/luajit.gyp:libluajit',
        'deps/yajl.gyp:yajl',
        'deps/yajl.gyp:copy_headers',
        'deps/cares/cares.gyp:cares',
        'deps/uv/uv.gyp:libuv',
        'deps/zlib/zlib.gyp:zlib',
        'deps/luacrypto.gyp:luacrypto',
      ],
      'export_dependent_settings': [
        'deps/http-parser/http_parser.gyp:http_parser',
        'deps/luajit.gyp:luajit',
        'deps/luajit.gyp:libluajit',
        'deps/yajl.gyp:yajl',
        'deps/cares/cares.gyp:cares',
        'deps/uv/uv.gyp:libuv',
        'deps/luacrypto.gyp:luacrypto',
      ],
      'conditions': [
        ['OS=="linux" or OS=="freebsd" or OS=="openbsd" or OS=="solaris"',
          {
            'cflags': [ '--std=c89' ],
            'defines': [ '_GNU_SOURCE' ]
          }
        ],
        ['"<(without_ssl)" == "false"',
          {
            'sources': [
              'src/luv_tls.c',
              'src/luv_tls_conn.c',
            ],
            'dependencies': [
              'deps/openssl/openssl.gyp:openssl'
            ],
            'export_dependent_settings': [
              'deps/openssl/openssl.gyp:openssl'
            ],
            'defines': [ 'USE_OPENSSL' ],
          }
        ],
      ],
      'sources': [
        'src/lconstants.c',
        'src/lenv.c',
        'src/lhttp_parser.c',
        'src/lyajl.c',
        'src/los.c',
        'src/luv.c',
        'src/luv_fs.c',
        'src/luv_fs_watcher.c',
        'src/luv_dns.c',
        'src/luv_debug.c',
        'src/luv_handle.c',
        'src/luv_misc.c',
        'src/luv_pipe.c',
        'src/luv_poll.c',
        'src/luv_process.c',
        'src/luv_signal.c',
        'src/luv_stream.c',
        'src/luv_tcp.c',
        'src/luv_timer.c',
        'src/luv_tty.c',
        'src/luv_udp.c',
        'src/luv_zlib.c',
        'src/luvit_init.c',
        'src/lyajl.c',
        'src/utils.c',
        'lib/luvit/buffer.lua',
        'lib/luvit/childprocess.lua',
        'lib/luvit/core.lua',
        'lib/luvit/dgram.lua',
        'lib/luvit/dns.lua',
        'lib/luvit/fiber.lua',
        'lib/luvit/fs.lua',
        'lib/luvit/http.lua',
        'lib/luvit/https.lua',
        'lib/luvit/json.lua',
        'lib/luvit/luvit.lua',
        'lib/luvit/mime.lua',
        'lib/luvit/module.lua',
        'lib/luvit/net.lua',
        'lib/luvit/path_base.lua',
        'lib/luvit/path.lua',
        'lib/luvit/querystring.lua',
        'lib/luvit/repl.lua',
        'lib/luvit/stack.lua',
        'lib/luvit/timer.lua',
        'lib/luvit/tls.lua',
        'lib/luvit/url.lua',
        'lib/luvit/utils.lua',
        'lib/luvit/uv.lua',
        'lib/luvit/zlib.lua',
      ],
      'defines': [
        'LUVIT_OS="<(OS)"',
        'LUVIT_VERSION="<!(git --git-dir .git describe --tags)"',
        'HTTP_VERSION="<!(git --git-dir deps/http-parser/.git describe --tags)"',
        'UV_VERSION="<!(git --git-dir deps/uv/.git describe --tags)"',
        'LUAJIT_VERSION="<!(git --git-dir deps/luajit/.git describe --tags)"',
        'YAJL_VERSIONISH="<!(git --git-dir deps/yajl/.git describe --tags)"',
        'BUNDLE=1',
        'CARES_STATICLIB'
      ],
      'include_dirs': [
        'src',
      ],
      'direct_dependent_settings': {
        'include_dirs': [
          'src',
        ]
      },
      'rules': [
        {
          'rule_name': 'jit_lua',
          'extension': 'lua',
          'outputs': [
            '<(SHARED_INTERMEDIATE_DIR)/generated/<(RULE_INPUT_ROOT)_jit.c'
          ],
          'action': [
            '<(PRODUCT_DIR)/luajit',
            '-b', '-g', '<(RULE_INPUT_PATH)',
            '<(SHARED_INTERMEDIATE_DIR)/generated/<(RULE_INPUT_ROOT)_jit.c',
          ],
          'process_outputs_as_sources': 1,
          'message': 'luajit <(RULE_INPUT_PATH)'
        }
      ],
    },
    {
      'target_name': 'luvit',
      'type': 'executable',
      'dependencies': [
        'libluvit',
      ],
      'sources': [
        'src/luvit_main.c',
        'src/luvit_exports.c',
      ],
      'msvs-settings': {
        'VCLinkerTool': {
          'SubSystem': 1, # /subsystem:console
        },
      },
      'conditions': [
        ['OS == "win"',
          {
            'libraries': [
              '-lgdi32.lib',
              '-luser32.lib'
            ],
          }
        ],
        ['OS == "mac" and target_arch == "x64"',
          {
            'xcode_settings': {
              'OTHER_LDFLAGS': ['-pagezero_size', '10000', '-image_base', '100000000']
            }
          }
        ],
        ['OS == "linux"',
          {
            'libraries': ['-ldl'],
          }
        ],
        ['OS=="linux" or OS=="freebsd" or OS=="openbsd" or OS=="solaris"',
          {
            'cflags': [ '--std=c89' ],
            'defines': [ '_GNU_SOURCE' ]
          }
        ],
        ['"<(without_ssl)" == "false"',
          {
            'defines': [ 'USE_OPENSSL' ],
          }
        ],
      ],
      'defines': [ 'BUNDLE=1' ]
    },
    {
      'target_name': 'vector_luvit',
      'product_name': 'vector',
      'product_extension': 'luvit',
      'product_prefix': '',
      'type': 'shared_library',
      'dependencies': [
        'libluvit',
      ],
      'sources': [
        'examples/native/vector.c'
      ],
      'conditions': [
        ['OS=="linux" or OS=="freebsd" or OS=="openbsd" or OS=="solaris"',
          {
            'cflags': [ '--std=c89' ],
            'defines': [ '_GNU_SOURCE' ]
          }
        ],
      ],
    },
    {
      'target_name': 'install',
      'type': 'none',
      'conditions': [
        ['OS!="win"',
          {
            'type': 'none',
            'copies': [
              {
                'destination': '<(luvit_prefix)/bin',
                'files': [
                  'out/Debug/luvit'
                ]
              },
              {
                'destination': '<(luvit_prefix)/lib/luvit',
                'files': [
                  'lib/luvit/buffer.lua',
                  'lib/luvit/childprocess.lua',
                  'lib/luvit/core.lua',
                  'lib/luvit/dns.lua',
                  'lib/luvit/fiber.lua',
                  'lib/luvit/fs.lua',
                  'lib/luvit/http.lua',
                  'lib/luvit/https.lua',
                  'lib/luvit/json.lua',
                  'lib/luvit/luvit.lua',
                  'lib/luvit/mime.lua',
                  'lib/luvit/module.lua',
                  'lib/luvit/net.lua',
                  'lib/luvit/path.lua',
                  'lib/luvit/querystring.lua',
                  'lib/luvit/repl.lua',
                  'lib/luvit/stack.lua',
                  'lib/luvit/timer.lua',
                  'lib/luvit/tls.lua',
                  'lib/luvit/url.lua',
                  'lib/luvit/utils.lua',
                  'lib/luvit/uv.lua',
                  'lib/luvit/zlib.lua'
                ]
              },
              {
                'destination': '<(luvit_prefix)/include/luvit/luajit',
                'files': [
                  'deps/luajit/src/lua.h',
                  'deps/luajit/src/lauxlib.h',
                  'deps/luajit/src/luaconf.h',
                  'deps/luajit/src/luajit.h',
                  'deps/luajit/src/lualib.h'
                ]
              },
              {
                'destination': '<(luvit_prefix)/include/luvit/http_parser',
                'files': [
                  'deps/http-parser/http_parser.h'
                ]
              },
              {
                'destination': '<(luvit_prefix)/include/luvit/uv',
                'files': [
                  'deps/uv/include/'
                ]
              },
              {
                'destination': '<(luvit_prefix)/include/luvit',
                'files': [
                  'src/lconstants.h',
                  'src/lenv.h',
                  'src/lhttp_parser.h',
                  'src/los.h',
                  'src/luv.h',
                  'src/luv_debug.h',
                  'src/luv_dns.h',
                  'src/luv_fs.h',
                  'src/luv_fs_watcher.h',
                  'src/luv_handle.h',
                  'src/luv_misc.h',
                  'src/luv_pipe.h',
                  'src/luv_portability.h',
                  'src/luv_process.h',
                  'src/luv_stream.h',
                  'src/luv_tcp.h',
                  'src/luv_timer.h',
                  'src/luv_tls.h',
                  'src/luv_tls_root_certs.h',
                  'src/luv_tty.h',
                  'src/luv_udp.h',
                  'src/luv_zlib.h',
                  'src/luvit.h',
                  'src/luvit_exports.h',
                  'src/luvit_init.h',
                  'src/lyajl.h',
                  'src/utils.h'
                ]
              }
            ]
          }
        ]
      ]
    }
  ],
}
<commit_msg>Fix the path where libuv headers are installed<commit_after>{
  'targets': [
    {'target_name': 'libluvit',
      'type': 'static_library',
      'dependencies': [
        'deps/http-parser/http_parser.gyp:http_parser',
        'deps/luajit.gyp:luajit',
        'deps/luajit.gyp:libluajit',
        'deps/yajl.gyp:yajl',
        'deps/yajl.gyp:copy_headers',
        'deps/cares/cares.gyp:cares',
        'deps/uv/uv.gyp:libuv',
        'deps/zlib/zlib.gyp:zlib',
        'deps/luacrypto.gyp:luacrypto',
      ],
      'export_dependent_settings': [
        'deps/http-parser/http_parser.gyp:http_parser',
        'deps/luajit.gyp:luajit',
        'deps/luajit.gyp:libluajit',
        'deps/yajl.gyp:yajl',
        'deps/cares/cares.gyp:cares',
        'deps/uv/uv.gyp:libuv',
        'deps/luacrypto.gyp:luacrypto',
      ],
      'conditions': [
        ['OS=="linux" or OS=="freebsd" or OS=="openbsd" or OS=="solaris"',
          {
            'cflags': [ '--std=c89' ],
            'defines': [ '_GNU_SOURCE' ]
          }
        ],
        ['"<(without_ssl)" == "false"',
          {
            'sources': [
              'src/luv_tls.c',
              'src/luv_tls_conn.c',
            ],
            'dependencies': [
              'deps/openssl/openssl.gyp:openssl'
            ],
            'export_dependent_settings': [
              'deps/openssl/openssl.gyp:openssl'
            ],
            'defines': [ 'USE_OPENSSL' ],
          }
        ],
      ],
      'sources': [
        'src/lconstants.c',
        'src/lenv.c',
        'src/lhttp_parser.c',
        'src/lyajl.c',
        'src/los.c',
        'src/luv.c',
        'src/luv_fs.c',
        'src/luv_fs_watcher.c',
        'src/luv_dns.c',
        'src/luv_debug.c',
        'src/luv_handle.c',
        'src/luv_misc.c',
        'src/luv_pipe.c',
        'src/luv_poll.c',
        'src/luv_process.c',
        'src/luv_signal.c',
        'src/luv_stream.c',
        'src/luv_tcp.c',
        'src/luv_timer.c',
        'src/luv_tty.c',
        'src/luv_udp.c',
        'src/luv_zlib.c',
        'src/luvit_init.c',
        'src/lyajl.c',
        'src/utils.c',
        'lib/luvit/buffer.lua',
        'lib/luvit/childprocess.lua',
        'lib/luvit/core.lua',
        'lib/luvit/dgram.lua',
        'lib/luvit/dns.lua',
        'lib/luvit/fiber.lua',
        'lib/luvit/fs.lua',
        'lib/luvit/http.lua',
        'lib/luvit/https.lua',
        'lib/luvit/json.lua',
        'lib/luvit/luvit.lua',
        'lib/luvit/mime.lua',
        'lib/luvit/module.lua',
        'lib/luvit/net.lua',
        'lib/luvit/path_base.lua',
        'lib/luvit/path.lua',
        'lib/luvit/querystring.lua',
        'lib/luvit/repl.lua',
        'lib/luvit/stack.lua',
        'lib/luvit/timer.lua',
        'lib/luvit/tls.lua',
        'lib/luvit/url.lua',
        'lib/luvit/utils.lua',
        'lib/luvit/uv.lua',
        'lib/luvit/zlib.lua',
      ],
      'defines': [
        'LUVIT_OS="<(OS)"',
        'LUVIT_VERSION="<!(git --git-dir .git describe --tags)"',
        'HTTP_VERSION="<!(git --git-dir deps/http-parser/.git describe --tags)"',
        'UV_VERSION="<!(git --git-dir deps/uv/.git describe --tags)"',
        'LUAJIT_VERSION="<!(git --git-dir deps/luajit/.git describe --tags)"',
        'YAJL_VERSIONISH="<!(git --git-dir deps/yajl/.git describe --tags)"',
        'BUNDLE=1',
        'CARES_STATICLIB'
      ],
      'include_dirs': [
        'src',
      ],
      'direct_dependent_settings': {
        'include_dirs': [
          'src',
        ]
      },
      'rules': [
        {
          'rule_name': 'jit_lua',
          'extension': 'lua',
          'outputs': [
            '<(SHARED_INTERMEDIATE_DIR)/generated/<(RULE_INPUT_ROOT)_jit.c'
          ],
          'action': [
            '<(PRODUCT_DIR)/luajit',
            '-b', '-g', '<(RULE_INPUT_PATH)',
            '<(SHARED_INTERMEDIATE_DIR)/generated/<(RULE_INPUT_ROOT)_jit.c',
          ],
          'process_outputs_as_sources': 1,
          'message': 'luajit <(RULE_INPUT_PATH)'
        }
      ],
    },
    {
      'target_name': 'luvit',
      'type': 'executable',
      'dependencies': [
        'libluvit',
      ],
      'sources': [
        'src/luvit_main.c',
        'src/luvit_exports.c',
      ],
      'msvs-settings': {
        'VCLinkerTool': {
          'SubSystem': 1, # /subsystem:console
        },
      },
      'conditions': [
        ['OS == "win"',
          {
            'libraries': [
              '-lgdi32.lib',
              '-luser32.lib'
            ],
          }
        ],
        ['OS == "mac" and target_arch == "x64"',
          {
            'xcode_settings': {
              'OTHER_LDFLAGS': ['-pagezero_size', '10000', '-image_base', '100000000']
            }
          }
        ],
        ['OS == "linux"',
          {
            'libraries': ['-ldl'],
          }
        ],
        ['OS=="linux" or OS=="freebsd" or OS=="openbsd" or OS=="solaris"',
          {
            'cflags': [ '--std=c89' ],
            'defines': [ '_GNU_SOURCE' ]
          }
        ],
        ['"<(without_ssl)" == "false"',
          {
            'defines': [ 'USE_OPENSSL' ],
          }
        ],
      ],
      'defines': [ 'BUNDLE=1' ]
    },
    {
      'target_name': 'vector_luvit',
      'product_name': 'vector',
      'product_extension': 'luvit',
      'product_prefix': '',
      'type': 'shared_library',
      'dependencies': [
        'libluvit',
      ],
      'sources': [
        'examples/native/vector.c'
      ],
      'conditions': [
        ['OS=="linux" or OS=="freebsd" or OS=="openbsd" or OS=="solaris"',
          {
            'cflags': [ '--std=c89' ],
            'defines': [ '_GNU_SOURCE' ]
          }
        ],
      ],
    },
    {
      'target_name': 'install',
      'type': 'none',
      'conditions': [
        ['OS!="win"',
          {
            'type': 'none',
            'copies': [
              {
                'destination': '<(luvit_prefix)/bin',
                'files': [
                  'out/Debug/luvit'
                ]
              },
              {
                'destination': '<(luvit_prefix)/lib/luvit',
                'files': [
                  'lib/luvit/buffer.lua',
                  'lib/luvit/childprocess.lua',
                  'lib/luvit/core.lua',
                  'lib/luvit/dns.lua',
                  'lib/luvit/fiber.lua',
                  'lib/luvit/fs.lua',
                  'lib/luvit/http.lua',
                  'lib/luvit/https.lua',
                  'lib/luvit/json.lua',
                  'lib/luvit/luvit.lua',
                  'lib/luvit/mime.lua',
                  'lib/luvit/module.lua',
                  'lib/luvit/net.lua',
                  'lib/luvit/path.lua',
                  'lib/luvit/querystring.lua',
                  'lib/luvit/repl.lua',
                  'lib/luvit/stack.lua',
                  'lib/luvit/timer.lua',
                  'lib/luvit/tls.lua',
                  'lib/luvit/url.lua',
                  'lib/luvit/utils.lua',
                  'lib/luvit/uv.lua',
                  'lib/luvit/zlib.lua'
                ]
              },
              {
                'destination': '<(luvit_prefix)/include/luvit/luajit',
                'files': [
                  'deps/luajit/src/lua.h',
                  'deps/luajit/src/lauxlib.h',
                  'deps/luajit/src/luaconf.h',
                  'deps/luajit/src/luajit.h',
                  'deps/luajit/src/lualib.h'
                ]
              },
              {
                'destination': '<(luvit_prefix)/include/luvit/http_parser',
                'files': [
                  'deps/http-parser/http_parser.h'
                ]
              },
              {
                'destination': '<(luvit_prefix)/include/luvit/uv',
                'files': [
                  'deps/uv/include/uv-private',
                  'deps/uv/include/uv.h'
                ]
              },
              {
                'destination': '<(luvit_prefix)/include/luvit',
                'files': [
                  'src/lconstants.h',
                  'src/lenv.h',
                  'src/lhttp_parser.h',
                  'src/los.h',
                  'src/luv.h',
                  'src/luv_debug.h',
                  'src/luv_dns.h',
                  'src/luv_fs.h',
                  'src/luv_fs_watcher.h',
                  'src/luv_handle.h',
                  'src/luv_misc.h',
                  'src/luv_pipe.h',
                  'src/luv_portability.h',
                  'src/luv_process.h',
                  'src/luv_stream.h',
                  'src/luv_tcp.h',
                  'src/luv_timer.h',
                  'src/luv_tls.h',
                  'src/luv_tls_root_certs.h',
                  'src/luv_tty.h',
                  'src/luv_udp.h',
                  'src/luv_zlib.h',
                  'src/luvit.h',
                  'src/luvit_exports.h',
                  'src/luvit_init.h',
                  'src/lyajl.h',
                  'src/utils.h'
                ]
              }
            ]
          }
        ]
      ]
    }
  ],
}
<|endoftext|>


<commit_before># -- coding: utf-8 --
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
""" General helper functions
for use in the configuration process

"""
import subprocess
import logging
import sys
import os.path
import re
import shutil
from netaddr import *

PUBLIC_INTERFACES = {"router": "eth2", "vpcrouter": "eth1"}

STATE_COMMANDS = {"router": "ip addr | grep eth0 | grep inet | wc -l | xargs bash -c  'if [ $0 == 2 ]; then echo \"MASTER\"; else echo \"BACKUP\"; fi'",
                  "vpcrouter": "ip addr | grep eth1 | grep state | awk '{print $9;}' | xargs bash -c 'if [ $0 == \"UP\" ]; then echo \"MASTER\"; else echo \"BACKUP\"; fi'"}

def reconfigure_interfaces(router_config, interfaces):
    for interface in interfaces:
        cmd = "ip link show %s | grep 'state DOWN'" % interface.get_device()
        for device in execute(cmd):
            if " DOWN " in device:
                cmd = "ip link set %s up" % interface.get_device()
                # If redundant only bring up public interfaces that are not eth1.
                # Reason: private gateways are public interfaces.
                # master.py and keepalived will deal with eth1 public interface.

                if router_config.is_redundant() and interface.is_public():
                    state_cmd = STATE_COMMANDS[router_config.get_type()]
                    logging.info("Check state command => %s" % state_cmd)
                    state = execute(state_cmd)[0]
                    logging.info("Route state => %s" % state)
                    if interface.get_device() != PUBLIC_INTERFACES[router_config.get_type()] and state == "MASTER":
                        execute(cmd)
                else:
                    execute(cmd)

def is_mounted(name):
    for i in execute("mount"):
        vals = i.lstrip().split()
        if vals[0] == "tmpfs" and vals[2] == name:
            return True
    return False

def mount_tmpfs(name):
    if not is_mounted(name):
        execute("mount tmpfs %s -t tmpfs" % name)

def umount_tmpfs(name):
    if is_mounted(name):
        execute("umount %s" % name)

def rm(name):
    os.remove(name) if os.path.isfile(name) else None

def rmdir(name):
    if name:
        shutil.rmtree(name, True)

def mkdir(name, mode, fatal):
    try:
        os.makedirs(name, mode)
    except OSError as e:
        if e.errno != 17:
            print "failed to make directories " + name + " due to :" + e.strerror
            if(fatal):
                sys.exit(1)

def updatefile(filename, val, mode):
    """ add val to file """
    handle = open(filename, 'r')
    for line in handle.read():
        if line.strip().lstrip() == val:
            return
    # set the value
    handle.close()
    handle = open(filename, mode)
    handle.write(val)
    handle.close()

def bool_to_yn(val):
    if val:
        return "yes"
    return "no"

def get_device_info():
    """ Returns all devices on system with their ipv4 ip netmask """
    list = []
    for i in execute("ip addr show"):
        vals = i.strip().lstrip().rstrip().split()
        if vals[0] == "inet":
            to = {}
            to['ip'] = vals[1]
            to['dev'] = vals[-1]
            to['network'] = IPNetwork(to['ip'])
            to['dnsmasq'] = False
            list.append(to)
    return list

def get_domain():
    for line in open("/etc/resolv.conf"):
        vals = line.lstrip().split()
        if vals[0] == "domain":
            return vals[1]
    return "cloudnine.internal"

def get_device(ip):
    """ Returns the device which has a specific ip
    If the ip is not found returns an empty string
    """
    for i in execute("ip addr show"):
        vals = i.strip().lstrip().rstrip().split()
        if vals[0] == "inet":
            if vals[1].split('/')[0] == ip:
                return vals[-1]
    return ""

def get_ip(device):
    """ Return first ip on an interface """
    cmd = "ip addr show dev %s" % device
    for i in execute(cmd):
        vals = i.lstrip().split()
        if (vals[0] == 'inet'):
            return vals[1]
    return ""

def definedinfile(filename, val):
    """ Check if val is defined in the file """
    for line in open(filename):
        if re.search(val, line):
            return True
    return False

def addifmissing(filename, val):
    """ Add something to a file
    if it is not already there """
    if not os.path.isfile(filename):
        logging.debug("File %s doesn't exist, so create" % filename)
        open(filename, "w").close()
    if not definedinfile(filename, val):
        updatefile(filename, val + "\n", "a")
        logging.debug("Added %s to file %s" % (val, filename))
        return True
    return False

def get_hostname():
    for line in open("/etc/hostname"):
        return line.strip()

def execute(command):
    """ Execute command """
    returncode = -1
    try:
        logging.info("Executing: %s" % command)
        result = subprocess.check_output(command, shell=True)
        returncode = 0
        return result.splitlines()
    except subprocess.CalledProcessError as e:
        logging.error(e)
        returncode = e.returncode
    finally:
        logging.debug("Executed: %s - exitstatus=%s " % (command, returncode))

    return list()

def save_iptables(command, iptables_file):
    """ Execute command """
    logging.debug("Saving iptables for %s" % command)

    result = execute(command)
    fIptables = open(iptables_file, "w+")

    for line in result:
        fIptables.write(line)
        fIptables.write("\n")
    fIptables.close()

def execute2(command):
    """ Execute command """
    logging.debug("Executing: %s" % command)
    p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
    p.wait()
    return p

def service(name, op):
    execute("systemctl %s %s" % (op, name))
    logging.info("Service %s %s" % (name, op))

def start_if_stopped(name):
    ret = execute2("systemctl is-active %s" % name)
    if ret.returncode:
        execute2("systemctl start %s" % name)

def hup_dnsmasq(name, user):
    pid = ""
    for i in execute("ps -ef | grep %s" % name):
        vals = i.lstrip().split()
        if (vals[0] == user):
            pid = vals[1]
    if pid:
        logging.info("Sent hup to %s", name)
        execute("kill -HUP %s" % pid)
    else:
        service("dnsmasq", "start")

def copy_if_needed(src, dest):
    """ Copy a file if the destination does not already exist
    """
    if os.path.isfile(dest):
        return
    copy(src, dest)

def copy(src, dest):
    """
    copy source to destination.
    """
    try:
        shutil.copy2(src, dest)
    except IOError:
        logging.error("Could not copy %s to %s" % (src, dest))
    else:
        logging.info("Copied %s to %s" % (src, dest))
<commit_msg>Log command output in CsHelper.execute command (#2568)<commit_after># -- coding: utf-8 --
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
""" General helper functions
for use in the configuration process

"""
import subprocess
import logging
import sys
import os.path
import re
import shutil
from netaddr import *

PUBLIC_INTERFACES = {"router": "eth2", "vpcrouter": "eth1"}

STATE_COMMANDS = {"router": "ip addr | grep eth0 | grep inet | wc -l | xargs bash -c  'if [ $0 == 2 ]; then echo \"MASTER\"; else echo \"BACKUP\"; fi'",
                  "vpcrouter": "ip addr | grep eth1 | grep state | awk '{print $9;}' | xargs bash -c 'if [ $0 == \"UP\" ]; then echo \"MASTER\"; else echo \"BACKUP\"; fi'"}

def reconfigure_interfaces(router_config, interfaces):
    for interface in interfaces:
        cmd = "ip link show %s | grep 'state DOWN'" % interface.get_device()
        for device in execute(cmd):
            if " DOWN " in device:
                cmd = "ip link set %s up" % interface.get_device()
                # If redundant only bring up public interfaces that are not eth1.
                # Reason: private gateways are public interfaces.
                # master.py and keepalived will deal with eth1 public interface.

                if router_config.is_redundant() and interface.is_public():
                    state_cmd = STATE_COMMANDS[router_config.get_type()]
                    logging.info("Check state command => %s" % state_cmd)
                    state = execute(state_cmd)[0]
                    logging.info("Route state => %s" % state)
                    if interface.get_device() != PUBLIC_INTERFACES[router_config.get_type()] and state == "MASTER":
                        execute(cmd)
                else:
                    execute(cmd)

def is_mounted(name):
    for i in execute("mount"):
        vals = i.lstrip().split()
        if vals[0] == "tmpfs" and vals[2] == name:
            return True
    return False

def mount_tmpfs(name):
    if not is_mounted(name):
        execute("mount tmpfs %s -t tmpfs" % name)

def umount_tmpfs(name):
    if is_mounted(name):
        execute("umount %s" % name)

def rm(name):
    os.remove(name) if os.path.isfile(name) else None

def rmdir(name):
    if name:
        shutil.rmtree(name, True)

def mkdir(name, mode, fatal):
    try:
        os.makedirs(name, mode)
    except OSError as e:
        if e.errno != 17:
            print "failed to make directories " + name + " due to :" + e.strerror
            if(fatal):
                sys.exit(1)

def updatefile(filename, val, mode):
    """ add val to file """
    handle = open(filename, 'r')
    for line in handle.read():
        if line.strip().lstrip() == val:
            return
    # set the value
    handle.close()
    handle = open(filename, mode)
    handle.write(val)
    handle.close()

def bool_to_yn(val):
    if val:
        return "yes"
    return "no"

def get_device_info():
    """ Returns all devices on system with their ipv4 ip netmask """
    list = []
    for i in execute("ip addr show"):
        vals = i.strip().lstrip().rstrip().split()
        if vals[0] == "inet":
            to = {}
            to['ip'] = vals[1]
            to['dev'] = vals[-1]
            to['network'] = IPNetwork(to['ip'])
            to['dnsmasq'] = False
            list.append(to)
    return list

def get_domain():
    for line in open("/etc/resolv.conf"):
        vals = line.lstrip().split()
        if vals[0] == "domain":
            return vals[1]
    return "cloudnine.internal"

def get_device(ip):
    """ Returns the device which has a specific ip
    If the ip is not found returns an empty string
    """
    for i in execute("ip addr show"):
        vals = i.strip().lstrip().rstrip().split()
        if vals[0] == "inet":
            if vals[1].split('/')[0] == ip:
                return vals[-1]
    return ""

def get_ip(device):
    """ Return first ip on an interface """
    cmd = "ip addr show dev %s" % device
    for i in execute(cmd):
        vals = i.lstrip().split()
        if (vals[0] == 'inet'):
            return vals[1]
    return ""

def definedinfile(filename, val):
    """ Check if val is defined in the file """
    for line in open(filename):
        if re.search(val, line):
            return True
    return False

def addifmissing(filename, val):
    """ Add something to a file
    if it is not already there """
    if not os.path.isfile(filename):
        logging.debug("File %s doesn't exist, so create" % filename)
        open(filename, "w").close()
    if not definedinfile(filename, val):
        updatefile(filename, val + "\n", "a")
        logging.debug("Added %s to file %s" % (val, filename))
        return True
    return False

def get_hostname():
    for line in open("/etc/hostname"):
        return line.strip()

def execute(command):
    """ Execute command """
    returncode = -1
    try:
        logging.info("Executing: %s" % command)
        result = subprocess.check_output(command, shell=True)
        returncode = 0

        logging.debug("Command [%s] has the result [%s]" % (command, result))
        return result.splitlines()
    except subprocess.CalledProcessError as e:
        logging.error(e)
        returncode = e.returncode
    finally:
        logging.debug("Executed: %s - exitstatus=%s " % (command, returncode))

    return list()

def save_iptables(command, iptables_file):
    """ Execute command """
    logging.debug("Saving iptables for %s" % command)

    result = execute(command)
    fIptables = open(iptables_file, "w+")

    for line in result:
        fIptables.write(line)
        fIptables.write("\n")
    fIptables.close()

def execute2(command):
    """ Execute command """
    logging.debug("Executing: %s" % command)
    p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
    p.wait()
    return p

def service(name, op):
    execute("systemctl %s %s" % (op, name))
    logging.info("Service %s %s" % (name, op))

def start_if_stopped(name):
    ret = execute2("systemctl is-active %s" % name)
    if ret.returncode:
        execute2("systemctl start %s" % name)

def hup_dnsmasq(name, user):
    pid = ""
    for i in execute("ps -ef | grep %s" % name):
        vals = i.lstrip().split()
        if (vals[0] == user):
            pid = vals[1]
    if pid:
        logging.info("Sent hup to %s", name)
        execute("kill -HUP %s" % pid)
    else:
        service("dnsmasq", "start")

def copy_if_needed(src, dest):
    """ Copy a file if the destination does not already exist
    """
    if os.path.isfile(dest):
        return
    copy(src, dest)

def copy(src, dest):
    """
    copy source to destination.
    """
    try:
        shutil.copy2(src, dest)
    except IOError:
        logging.error("Could not copy %s to %s" % (src, dest))
    else:
        logging.info("Copied %s to %s" % (src, dest))
<|endoftext|>


<commit_before>"""Support for AdGuard Home switches."""
from __future__ import annotations

from datetime import timedelta
import logging
from typing import Any

from adguardhome import AdGuardHome, AdGuardHomeConnectionError, AdGuardHomeError

from homeassistant.components.switch import SwitchEntity
from homeassistant.config_entries import ConfigEntry
from homeassistant.core import HomeAssistant
from homeassistant.exceptions import PlatformNotReady
from homeassistant.helpers.entity_platform import AddEntitiesCallback

from . import AdGuardHomeDeviceEntity
from .const import DATA_ADGUARD_CLIENT, DATA_ADGUARD_VERSION, DOMAIN

_LOGGER = logging.getLogger(__name__)

SCAN_INTERVAL = timedelta(seconds=10)
PARALLEL_UPDATES = 1

async def async_setup_entry(
    hass: HomeAssistant,
    entry: ConfigEntry,
    async_add_entities: AddEntitiesCallback,
) -> None:
    """Set up AdGuard Home switch based on a config entry."""
    adguard = hass.data[DOMAIN][entry.entry_id][DATA_ADGUARD_CLIENT]

    try:
        version = await adguard.version()
    except AdGuardHomeConnectionError as exception:
        raise PlatformNotReady from exception

    hass.data[DOMAIN][entry.entry_id][DATA_ADGUARD_VERSION] = version

    switches = [
        AdGuardHomeProtectionSwitch(adguard, entry),
        AdGuardHomeFilteringSwitch(adguard, entry),
        AdGuardHomeParentalSwitch(adguard, entry),
        AdGuardHomeSafeBrowsingSwitch(adguard, entry),
        AdGuardHomeSafeSearchSwitch(adguard, entry),
        AdGuardHomeQueryLogSwitch(adguard, entry),
    ]
    async_add_entities(switches, True)

class AdGuardHomeSwitch(AdGuardHomeDeviceEntity, SwitchEntity):
    """Defines a AdGuard Home switch."""

    def __init__(
        self,
        adguard: AdGuardHome,
        entry: ConfigEntry,
        name: str,
        icon: str,
        key: str,
        enabled_default: bool = True,
    ) -> None:
        """Initialize AdGuard Home switch."""
        self._state = False
        self._key = key
        super().__init__(adguard, entry, name, icon, enabled_default)

    @property
    def unique_id(self) -> str:
        """Return the unique ID for this sensor."""
        return "_".join(
            [DOMAIN, self.adguard.host, str(self.adguard.port), "switch", self._key]
        )

    @property
    def is_on(self) -> bool:
        """Return the state of the switch."""
        return self._state

    async def async_turn_off(self, **kwargs: Any) -> None:
        """Turn off the switch."""
        try:
            await self._adguard_turn_off()
        except AdGuardHomeError:
            _LOGGER.error("An error occurred while turning off AdGuard Home switch")
            self._available = False

    async def _adguard_turn_off(self) -> None:
        """Turn off the switch."""
        raise NotImplementedError()

    async def async_turn_on(self, **kwargs: Any) -> None:
        """Turn on the switch."""
        try:
            await self._adguard_turn_on()
        except AdGuardHomeError:
            _LOGGER.error("An error occurred while turning on AdGuard Home switch")
            self._available = False

    async def _adguard_turn_on(self) -> None:
        """Turn on the switch."""
        raise NotImplementedError()

class AdGuardHomeProtectionSwitch(AdGuardHomeSwitch):
    """Defines a AdGuard Home protection switch."""

    def __init__(self, adguard: AdGuardHome, entry: ConfigEntry) -> None:
        """Initialize AdGuard Home switch."""
        super().__init__(adguard, entry, "Protection", "mdi:shield-check", "protection")

    async def _adguard_turn_off(self) -> None:
        """Turn off the switch."""
        await self.adguard.disable_protection()

    async def _adguard_turn_on(self) -> None:
        """Turn on the switch."""
        await self.adguard.enable_protection()

    async def _adguard_update(self) -> None:
        """Update AdGuard Home entity."""
        self._state = await self.adguard.protection_enabled()

class AdGuardHomeParentalSwitch(AdGuardHomeSwitch):
    """Defines a AdGuard Home parental control switch."""

    def __init__(self, adguard: AdGuardHome, entry: ConfigEntry) -> None:
        """Initialize AdGuard Home switch."""
        super().__init__(
            adguard, entry, "Parental control", "mdi:shield-check", "parental"
        )

    async def _adguard_turn_off(self) -> None:
        """Turn off the switch."""
        await self.adguard.parental.disable()

    async def _adguard_turn_on(self) -> None:
        """Turn on the switch."""
        await self.adguard.parental.enable()

    async def _adguard_update(self) -> None:
        """Update AdGuard Home entity."""
        self._state = await self.adguard.parental.enabled()

class AdGuardHomeSafeSearchSwitch(AdGuardHomeSwitch):
    """Defines a AdGuard Home safe search switch."""

    def __init__(self, adguard: AdGuardHome, entry: ConfigEntry) -> None:
        """Initialize AdGuard Home switch."""
        super().__init__(
            adguard, entry, "Safe search", "mdi:shield-check", "safesearch"
        )

    async def _adguard_turn_off(self) -> None:
        """Turn off the switch."""
        await self.adguard.safesearch.disable()

    async def _adguard_turn_on(self) -> None:
        """Turn on the switch."""
        await self.adguard.safesearch.enable()

    async def _adguard_update(self) -> None:
        """Update AdGuard Home entity."""
        self._state = await self.adguard.safesearch.enabled()

class AdGuardHomeSafeBrowsingSwitch(AdGuardHomeSwitch):
    """Defines a AdGuard Home safe search switch."""

    def __init__(self, adguard: AdGuardHome, entry: ConfigEntry) -> None:
        """Initialize AdGuard Home switch."""
        super().__init__(
            adguard, entry, "Safe browsing", "mdi:shield-check", "safebrowsing"
        )

    async def _adguard_turn_off(self) -> None:
        """Turn off the switch."""
        await self.adguard.safebrowsing.disable()

    async def _adguard_turn_on(self) -> None:
        """Turn on the switch."""
        await self.adguard.safebrowsing.enable()

    async def _adguard_update(self) -> None:
        """Update AdGuard Home entity."""
        self._state = await self.adguard.safebrowsing.enabled()

class AdGuardHomeFilteringSwitch(AdGuardHomeSwitch):
    """Defines a AdGuard Home filtering switch."""

    def __init__(self, adguard: AdGuardHome, entry: ConfigEntry) -> None:
        """Initialize AdGuard Home switch."""
        super().__init__(adguard, entry, "Filtering", "mdi:shield-check", "filtering")

    async def _adguard_turn_off(self) -> None:
        """Turn off the switch."""
        await self.adguard.filtering.disable()

    async def _adguard_turn_on(self) -> None:
        """Turn on the switch."""
        await self.adguard.filtering.enable()

    async def _adguard_update(self) -> None:
        """Update AdGuard Home entity."""
        self._state = await self.adguard.filtering.enabled()

class AdGuardHomeQueryLogSwitch(AdGuardHomeSwitch):
    """Defines a AdGuard Home query log switch."""

    def __init__(self, adguard: AdGuardHome, entry: ConfigEntry) -> None:
        """Initialize AdGuard Home switch."""
        super().__init__(
            adguard,
            entry,
            "Query log",
            "mdi:shield-check",
            "querylog",
            enabled_default=False,
        )

    async def _adguard_turn_off(self) -> None:
        """Turn off the switch."""
        await self.adguard.querylog.disable()

    async def _adguard_turn_on(self) -> None:
        """Turn on the switch."""
        await self.adguard.querylog.enable()

    async def _adguard_update(self) -> None:
        """Update AdGuard Home entity."""
        self._state = await self.adguard.querylog.enabled()
<commit_msg>Add entity descriptions in AdGuard Home switches (#75229)<commit_after>"""Support for AdGuard Home switches."""
from __future__ import annotations

from collections.abc import Callable, Coroutine
from dataclasses import dataclass
from datetime import timedelta
import logging
from typing import Any

from adguardhome import AdGuardHome, AdGuardHomeConnectionError, AdGuardHomeError

from homeassistant.components.switch import SwitchEntity, SwitchEntityDescription
from homeassistant.config_entries import ConfigEntry
from homeassistant.core import HomeAssistant
from homeassistant.exceptions import PlatformNotReady
from homeassistant.helpers.entity_platform import AddEntitiesCallback

from . import AdGuardHomeDeviceEntity
from .const import DATA_ADGUARD_CLIENT, DATA_ADGUARD_VERSION, DOMAIN

_LOGGER = logging.getLogger(__name__)

SCAN_INTERVAL = timedelta(seconds=10)
PARALLEL_UPDATES = 1

@dataclass
class AdGuardHomeSwitchEntityDescriptionMixin:
    """Mixin for required keys."""

    is_on_fn: Callable[[AdGuardHome], Callable[[], Coroutine[Any, Any, bool]]]
    turn_on_fn: Callable[[AdGuardHome], Callable[[], Coroutine[Any, Any, None]]]
    turn_off_fn: Callable[[AdGuardHome], Callable[[], Coroutine[Any, Any, None]]]

@dataclass
class AdGuardHomeSwitchEntityDescription(
    SwitchEntityDescription, AdGuardHomeSwitchEntityDescriptionMixin
):
    """Describes AdGuard Home switch entity."""

SWITCHES: tuple[AdGuardHomeSwitchEntityDescription, ...] = (
    AdGuardHomeSwitchEntityDescription(
        key="protection",
        name="Protection",
        icon="mdi:shield-check",
        is_on_fn=lambda adguard: adguard.protection_enabled,
        turn_on_fn=lambda adguard: adguard.enable_protection,
        turn_off_fn=lambda adguard: adguard.disable_protection,
    ),
    AdGuardHomeSwitchEntityDescription(
        key="parental",
        name="Parental control",
        icon="mdi:shield-check",
        is_on_fn=lambda adguard: adguard.parental.enabled,
        turn_on_fn=lambda adguard: adguard.parental.enable,
        turn_off_fn=lambda adguard: adguard.parental.disable,
    ),
    AdGuardHomeSwitchEntityDescription(
        key="safesearch",
        name="Safe search",
        icon="mdi:shield-check",
        is_on_fn=lambda adguard: adguard.safesearch.enabled,
        turn_on_fn=lambda adguard: adguard.safesearch.enable,
        turn_off_fn=lambda adguard: adguard.safesearch.disable,
    ),
    AdGuardHomeSwitchEntityDescription(
        key="safebrowsing",
        name="Safe browsing",
        icon="mdi:shield-check",
        is_on_fn=lambda adguard: adguard.safebrowsing.enabled,
        turn_on_fn=lambda adguard: adguard.safebrowsing.enable,
        turn_off_fn=lambda adguard: adguard.safebrowsing.disable,
    ),
    AdGuardHomeSwitchEntityDescription(
        key="filtering",
        name="Filtering",
        icon="mdi:shield-check",
        is_on_fn=lambda adguard: adguard.filtering.enabled,
        turn_on_fn=lambda adguard: adguard.filtering.enable,
        turn_off_fn=lambda adguard: adguard.filtering.disable,
    ),
    AdGuardHomeSwitchEntityDescription(
        key="querylog",
        name="Query log",
        icon="mdi:shield-check",
        is_on_fn=lambda adguard: adguard.querylog.enabled,
        turn_on_fn=lambda adguard: adguard.querylog.enable,
        turn_off_fn=lambda adguard: adguard.querylog.disable,
    ),
)

async def async_setup_entry(
    hass: HomeAssistant,
    entry: ConfigEntry,
    async_add_entities: AddEntitiesCallback,
) -> None:
    """Set up AdGuard Home switch based on a config entry."""
    adguard = hass.data[DOMAIN][entry.entry_id][DATA_ADGUARD_CLIENT]

    try:
        version = await adguard.version()
    except AdGuardHomeConnectionError as exception:
        raise PlatformNotReady from exception

    hass.data[DOMAIN][entry.entry_id][DATA_ADGUARD_VERSION] = version

    async_add_entities(
        [AdGuardHomeSwitch(adguard, entry, description) for description in SWITCHES],
        True,
    )

class AdGuardHomeSwitch(AdGuardHomeDeviceEntity, SwitchEntity):
    """Defines a AdGuard Home switch."""

    entity_description: AdGuardHomeSwitchEntityDescription

    def __init__(
        self,
        adguard: AdGuardHome,
        entry: ConfigEntry,
        description: AdGuardHomeSwitchEntityDescription,
    ) -> None:
        """Initialize AdGuard Home switch."""
        self.entity_description = description

        self._attr_unique_id = "_".join(
            [DOMAIN, adguard.host, str(adguard.port), "switch", description.key]
        )

        super().__init__(
            adguard,
            entry,
            description.name,
            description.icon,
            description.entity_registry_enabled_default,
        )

    async def async_turn_off(self, **kwargs: Any) -> None:
        """Turn off the switch."""
        try:
            await self.entity_description.turn_off_fn(self.adguard)()
        except AdGuardHomeError:
            _LOGGER.error("An error occurred while turning off AdGuard Home switch")
            self._available = False

    async def async_turn_on(self, **kwargs: Any) -> None:
        """Turn on the switch."""
        try:
            await self.entity_description.turn_on_fn(self.adguard)()
        except AdGuardHomeError:
            _LOGGER.error("An error occurred while turning on AdGuard Home switch")
            self._available = False

    async def _adguard_update(self) -> None:
        """Update AdGuard Home entity."""
        self._attr_is_on = await self.entity_description.is_on_fn(self.adguard)()
<|endoftext|>


<commit_before>"""Backup manager for the Backup integration."""
from __future__ import annotations

from dataclasses import asdict, dataclass
import hashlib
import json
from pathlib import Path
import tarfile
from tarfile import TarError
from tempfile import TemporaryDirectory

from securetar import SecureTarFile, atomic_contents_add

from homeassistant.const import __version__ as HAVERSION
from homeassistant.core import HomeAssistant
from homeassistant.exceptions import HomeAssistantError
from homeassistant.util import dt, json as json_util

from .const import EXCLUDE_FROM_BACKUP, LOGGER

@dataclass
class Backup:
    """Backup class."""

    slug: str
    name: str
    date: str
    path: Path
    size: float

    def as_dict(self) -> dict:
        """Return a dict representation of this backup."""
        return {**asdict(self), "path": self.path.as_posix()}

class BackupManager:
    """Backup manager for the Backup integration."""

    def __init__(self, hass: HomeAssistant) -> None:
        """Initialize the backup manager."""
        self.hass = hass
        self.backup_dir = Path(hass.config.path("backups"))
        self.backing_up = False
        self.backups: dict[str, Backup] = {}
        self.loaded = False

    async def load_backups(self) -> None:
        """Load data of stored backup files."""
        backups = {}

        def _read_backups() -> None:
            for backup_path in self.backup_dir.glob("*.tar"):
                try:
                    with tarfile.open(backup_path, "r:") as backup_file:
                        if data_file := backup_file.extractfile("./backup.json"):
                            data = json.loads(data_file.read())
                            backup = Backup(
                                slug=data["slug"],
                                name=data["name"],
                                date=data["date"],
                                path=backup_path,
                                size=round(backup_path.stat().st_size / 1_048_576, 2),
                            )
                            backups[backup.slug] = backup
                except (OSError, TarError, json.JSONDecodeError) as err:
                    LOGGER.warning("Unable to read backup %s: %s", backup_path, err)

        await self.hass.async_add_executor_job(_read_backups)
        LOGGER.debug("Loaded %s backups", len(backups))
        self.backups = backups
        self.loaded = True

    async def get_backups(self) -> dict[str, Backup]:
        """Return backups."""
        if not self.loaded:
            await self.load_backups()

        return self.backups

    async def get_backup(self, slug: str) -> Backup | None:
        """Return a backup."""
        if not self.loaded:
            await self.load_backups()

        if not (backup := self.backups.get(slug)):
            return None

        if not backup.path.exists():
            LOGGER.debug(
                "Removing tracked backup (%s) that does not exists on the expected path %s",
                backup.slug,
                backup.path,
            )
            self.backups.pop(slug)
            return None

        return backup

    async def remove_backup(self, slug: str) -> None:
        """Remove a backup."""
        if (backup := await self.get_backup(slug)) is None:
            return

        await self.hass.async_add_executor_job(backup.path.unlink, True)
        LOGGER.debug("Removed backup located at %s", backup.path)
        self.backups.pop(slug)

    async def generate_backup(self) -> Backup:
        """Generate a backup."""
        if self.backing_up:
            raise HomeAssistantError("Backup already in progress")

        try:
            self.backing_up = True
            backup_name = f"Core {HAVERSION}"
            date_str = dt.now().isoformat()
            slug = _generate_slug(date_str, backup_name)

            backup_data = {
                "slug": slug,
                "name": backup_name,
                "date": date_str,
                "type": "partial",
                "folders": ["homeassistant"],
                "homeassistant": {"version": HAVERSION},
                "compressed": True,
            }
            tar_file_path = Path(self.backup_dir, f"{slug}.tar")

            if not self.backup_dir.exists():
                LOGGER.debug("Creating backup directory")
                self.hass.async_add_executor_job(self.backup_dir.mkdir)

            def _create_backup() -> None:
                with TemporaryDirectory() as tmp_dir:
                    tmp_dir_path = Path(tmp_dir)
                    json_util.save_json(
                        tmp_dir_path.joinpath("./backup.json").as_posix(),
                        backup_data,
                    )
                    with SecureTarFile(tar_file_path, "w", gzip=False) as tar_file:
                        with SecureTarFile(
                            tmp_dir_path.joinpath("./homeassistant.tar.gz").as_posix(),
                            "w",
                        ) as core_tar:
                            atomic_contents_add(
                                tar_file=core_tar,
                                origin_path=Path(self.hass.config.path()),
                                excludes=EXCLUDE_FROM_BACKUP,
                                arcname="data",
                            )
                        tar_file.add(tmp_dir_path, arcname=".")

            await self.hass.async_add_executor_job(_create_backup)
            backup = Backup(
                slug=slug,
                name=backup_name,
                date=date_str,
                path=tar_file_path,
                size=round(tar_file_path.stat().st_size / 1_048_576, 2),
            )
            if self.loaded:
                self.backups[slug] = backup
            LOGGER.debug("Generated new backup with slug %s", slug)
            return backup
        finally:
            self.backing_up = False

def _generate_slug(date: str, name: str) -> str:
    """Generate a backup slug."""
    return hashlib.sha1(f"{date} - {name}".lower().encode()).hexdigest()[:8]
<commit_msg>Split out sync functions in backup manager (#67428)<commit_after>"""Backup manager for the Backup integration."""
from __future__ import annotations

from dataclasses import asdict, dataclass
import hashlib
import json
from pathlib import Path
import tarfile
from tarfile import TarError
from tempfile import TemporaryDirectory
from typing import Any

from securetar import SecureTarFile, atomic_contents_add

from homeassistant.const import __version__ as HAVERSION
from homeassistant.core import HomeAssistant
from homeassistant.exceptions import HomeAssistantError
from homeassistant.util import dt, json as json_util

from .const import EXCLUDE_FROM_BACKUP, LOGGER

@dataclass
class Backup:
    """Backup class."""

    slug: str
    name: str
    date: str
    path: Path
    size: float

    def as_dict(self) -> dict:
        """Return a dict representation of this backup."""
        return {**asdict(self), "path": self.path.as_posix()}

class BackupManager:
    """Backup manager for the Backup integration."""

    def __init__(self, hass: HomeAssistant) -> None:
        """Initialize the backup manager."""
        self.hass = hass
        self.backup_dir = Path(hass.config.path("backups"))
        self.backing_up = False
        self.backups: dict[str, Backup] = {}
        self.loaded = False

    async def load_backups(self) -> None:
        """Load data of stored backup files."""
        backups = await self.hass.async_add_executor_job(self._read_backups)
        LOGGER.debug("Loaded %s backups", len(backups))
        self.backups = backups
        self.loaded = True

    def _read_backups(self) -> dict[str, Backup]:
        """Read backups from disk."""
        backups: dict[str, Backup] = {}
        for backup_path in self.backup_dir.glob("*.tar"):
            try:
                with tarfile.open(backup_path, "r:") as backup_file:
                    if data_file := backup_file.extractfile("./backup.json"):
                        data = json.loads(data_file.read())
                        backup = Backup(
                            slug=data["slug"],
                            name=data["name"],
                            date=data["date"],
                            path=backup_path,
                            size=round(backup_path.stat().st_size / 1_048_576, 2),
                        )
                        backups[backup.slug] = backup
            except (OSError, TarError, json.JSONDecodeError) as err:
                LOGGER.warning("Unable to read backup %s: %s", backup_path, err)
        return backups

    async def get_backups(self) -> dict[str, Backup]:
        """Return backups."""
        if not self.loaded:
            await self.load_backups()

        return self.backups

    async def get_backup(self, slug: str) -> Backup | None:
        """Return a backup."""
        if not self.loaded:
            await self.load_backups()

        if not (backup := self.backups.get(slug)):
            return None

        if not backup.path.exists():
            LOGGER.debug(
                "Removing tracked backup (%s) that does not exists on the expected path %s",
                backup.slug,
                backup.path,
            )
            self.backups.pop(slug)
            return None

        return backup

    async def remove_backup(self, slug: str) -> None:
        """Remove a backup."""
        if (backup := await self.get_backup(slug)) is None:
            return

        await self.hass.async_add_executor_job(backup.path.unlink, True)
        LOGGER.debug("Removed backup located at %s", backup.path)
        self.backups.pop(slug)

    async def generate_backup(self) -> Backup:
        """Generate a backup."""
        if self.backing_up:
            raise HomeAssistantError("Backup already in progress")

        try:
            self.backing_up = True
            backup_name = f"Core {HAVERSION}"
            date_str = dt.now().isoformat()
            slug = _generate_slug(date_str, backup_name)

            backup_data = {
                "slug": slug,
                "name": backup_name,
                "date": date_str,
                "type": "partial",
                "folders": ["homeassistant"],
                "homeassistant": {"version": HAVERSION},
                "compressed": True,
            }
            tar_file_path = Path(self.backup_dir, f"{backup_data['slug']}.tar")

            if not self.backup_dir.exists():
                LOGGER.debug("Creating backup directory")
                self.hass.async_add_executor_job(self.backup_dir.mkdir)

            await self.hass.async_add_executor_job(
                self._generate_backup_contents,
                tar_file_path,
                backup_data,
            )
            backup = Backup(
                slug=slug,
                name=backup_name,
                date=date_str,
                path=tar_file_path,
                size=round(tar_file_path.stat().st_size / 1_048_576, 2),
            )
            if self.loaded:
                self.backups[slug] = backup
            LOGGER.debug("Generated new backup with slug %s", slug)
            return backup
        finally:
            self.backing_up = False

    def _generate_backup_contents(
        self,
        tar_file_path: Path,
        backup_data: dict[str, Any],
    ) -> None:
        """Generate backup contents."""
        with TemporaryDirectory() as tmp_dir, SecureTarFile(
            tar_file_path, "w", gzip=False
        ) as tar_file:
            tmp_dir_path = Path(tmp_dir)
            json_util.save_json(
                tmp_dir_path.joinpath("./backup.json").as_posix(),
                backup_data,
            )
            with SecureTarFile(
                tmp_dir_path.joinpath("./homeassistant.tar.gz").as_posix(),
                "w",
            ) as core_tar:
                atomic_contents_add(
                    tar_file=core_tar,
                    origin_path=Path(self.hass.config.path()),
                    excludes=EXCLUDE_FROM_BACKUP,
                    arcname="data",
                )
            tar_file.add(tmp_dir_path, arcname=".")

def _generate_slug(date: str, name: str) -> str:
    """Generate a backup slug."""
    return hashlib.sha1(f"{date} - {name}".lower().encode()).hexdigest()[:8]
<|endoftext|>


<commit_before>"""
Support for MQTT discovery.

For more details about this component, please refer to the documentation at
https://home-assistant.io/components/mqtt/#discovery
"""
import asyncio
import json
import logging
import re

from homeassistant.components import mqtt
from homeassistant.components.mqtt import ATTR_DISCOVERY_HASH, CONF_STATE_TOPIC
from homeassistant.const import CONF_DEVICE, CONF_PLATFORM
from homeassistant.helpers.discovery import async_load_platform
from homeassistant.helpers.dispatcher import async_dispatcher_send
from homeassistant.helpers.typing import HomeAssistantType

_LOGGER = logging.getLogger(__name__)

TOPIC_MATCHER = re.compile(
    r'(?P<prefix_topic>\w+)/(?P<component>\w+)/'
    r'(?:(?P<node_id>[a-zA-Z0-9_-]+)/)?(?P<object_id>[a-zA-Z0-9_-]+)/config')

SUPPORTED_COMPONENTS = [
    'binary_sensor', 'camera', 'cover', 'fan',
    'light', 'sensor', 'switch', 'lock', 'climate',
    'alarm_control_panel', 'vacuum']

CONFIG_ENTRY_COMPONENTS = [
    'binary_sensor',
    'camera',
    'cover',
    'light',
    'lock',
    'sensor',
    'switch',
    'climate',
    'alarm_control_panel',
    'fan',
    'vacuum',
]

DEPRECATED_PLATFORM_TO_SCHEMA = {
    'light': {
        'mqtt_json': 'json',
        'mqtt_template': 'template',
    }
}

ALREADY_DISCOVERED = 'mqtt_discovered_components'
DATA_CONFIG_ENTRY_LOCK = 'mqtt_config_entry_lock'
CONFIG_ENTRY_IS_SETUP = 'mqtt_config_entry_is_setup'
MQTT_DISCOVERY_UPDATED = 'mqtt_discovery_updated_{}'
MQTT_DISCOVERY_NEW = 'mqtt_discovery_new_{}_{}'

TOPIC_BASE = '~'

ABBREVIATIONS = {
    'aux_cmd_t': 'aux_command_topic',
    'aux_stat_tpl': 'aux_state_template',
    'aux_stat_t': 'aux_state_topic',
    'avty_t': 'availability_topic',
    'away_mode_cmd_t': 'away_mode_command_topic',
    'away_mode_stat_tpl': 'away_mode_state_template',
    'away_mode_stat_t': 'away_mode_state_topic',
    'bri_cmd_t': 'brightness_command_topic',
    'bri_scl': 'brightness_scale',
    'bri_stat_t': 'brightness_state_topic',
    'bri_val_tpl': 'brightness_value_template',
    'clr_temp_cmd_tpl': 'color_temp_command_template',
    'bat_lev_t': 'battery_level_topic',
    'bat_lev_tpl': 'battery_level_template',
    'chrg_t': 'charging_topic',
    'chrg_tpl': 'charging_template',
    'clr_temp_cmd_t': 'color_temp_command_topic',
    'clr_temp_stat_t': 'color_temp_state_topic',
    'clr_temp_val_tpl': 'color_temp_value_template',
    'cln_t': 'cleaning_topic',
    'cln_tpl': 'cleaning_template',
    'cmd_t': 'command_topic',
    'curr_temp_t': 'current_temperature_topic',
    'dev': 'device',
    'dev_cla': 'device_class',
    'dock_t': 'docked_topic',
    'dock_tpl': 'docked_template',
    'err_t': 'error_topic',
    'err_tpl': 'error_template',
    'fanspd_t': 'fan_speed_topic',
    'fanspd_tpl': 'fan_speed_template',
    'fanspd_lst': 'fan_speed_list',
    'fx_cmd_t': 'effect_command_topic',
    'fx_list': 'effect_list',
    'fx_stat_t': 'effect_state_topic',
    'fx_val_tpl': 'effect_value_template',
    'exp_aft': 'expire_after',
    'fan_mode_cmd_t': 'fan_mode_command_topic',
    'fan_mode_stat_tpl': 'fan_mode_state_template',
    'fan_mode_stat_t': 'fan_mode_state_topic',
    'frc_upd': 'force_update',
    'hold_cmd_t': 'hold_command_topic',
    'hold_stat_tpl': 'hold_state_template',
    'hold_stat_t': 'hold_state_topic',
    'ic': 'icon',
    'init': 'initial',
    'json_attr': 'json_attributes',
    'json_attr_t': 'json_attributes_topic',
    'max_temp': 'max_temp',
    'min_temp': 'min_temp',
    'mode_cmd_t': 'mode_command_topic',
    'mode_stat_tpl': 'mode_state_template',
    'mode_stat_t': 'mode_state_topic',
    'name': 'name',
    'on_cmd_type': 'on_command_type',
    'opt': 'optimistic',
    'osc_cmd_t': 'oscillation_command_topic',
    'osc_stat_t': 'oscillation_state_topic',
    'osc_val_tpl': 'oscillation_value_template',
    'pl_arm_away': 'payload_arm_away',
    'pl_arm_home': 'payload_arm_home',
    'pl_avail': 'payload_available',
    'pl_cls': 'payload_close',
    'pl_disarm': 'payload_disarm',
    'pl_hi_spd': 'payload_high_speed',
    'pl_lock': 'payload_lock',
    'pl_lo_spd': 'payload_low_speed',
    'pl_med_spd': 'payload_medium_speed',
    'pl_not_avail': 'payload_not_available',
    'pl_off': 'payload_off',
    'pl_on': 'payload_on',
    'pl_open': 'payload_open',
    'pl_osc_off': 'payload_oscillation_off',
    'pl_osc_on': 'payload_oscillation_on',
    'pl_stop': 'payload_stop',
    'pl_unlk': 'payload_unlock',
    'pow_cmd_t': 'power_command_topic',
    'ret': 'retain',
    'rgb_cmd_tpl': 'rgb_command_template',
    'rgb_cmd_t': 'rgb_command_topic',
    'rgb_stat_t': 'rgb_state_topic',
    'rgb_val_tpl': 'rgb_value_template',
    'send_cmd_t': 'send_command_topic',
    'send_if_off': 'send_if_off',
    'set_pos_tpl': 'set_position_template',
    'set_pos_t': 'set_position_topic',
    'spd_cmd_t': 'speed_command_topic',
    'spd_stat_t': 'speed_state_topic',
    'spd_val_tpl': 'speed_value_template',
    'spds': 'speeds',
    'stat_clsd': 'state_closed',
    'stat_off': 'state_off',
    'stat_on': 'state_on',
    'stat_open': 'state_open',
    'stat_t': 'state_topic',
    'stat_val_tpl': 'state_value_template',
    'sup_feat': 'supported_features',
    'swing_mode_cmd_t': 'swing_mode_command_topic',
    'swing_mode_stat_tpl': 'swing_mode_state_template',
    'swing_mode_stat_t': 'swing_mode_state_topic',
    'temp_cmd_t': 'temperature_command_topic',
    'temp_stat_tpl': 'temperature_state_template',
    'temp_stat_t': 'temperature_state_topic',
    'tilt_clsd_val': 'tilt_closed_value',
    'tilt_cmd_t': 'tilt_command_topic',
    'tilt_inv_stat': 'tilt_invert_state',
    'tilt_max': 'tilt_max',
    'tilt_min': 'tilt_min',
    'tilt_opnd_val': 'tilt_opened_value',
    'tilt_status_opt': 'tilt_status_optimistic',
    'tilt_status_t': 'tilt_status_topic',
    't': 'topic',
    'uniq_id': 'unique_id',
    'unit_of_meas': 'unit_of_measurement',
    'val_tpl': 'value_template',
    'whit_val_cmd_t': 'white_value_command_topic',
    'whit_val_scl': 'white_value_scale',
    'whit_val_stat_t': 'white_value_state_topic',
    'whit_val_tpl': 'white_value_template',
    'xy_cmd_t': 'xy_command_topic',
    'xy_stat_t': 'xy_state_topic',
    'xy_val_tpl': 'xy_value_template',
}

DEVICE_ABBREVIATIONS = {
    'cns': 'connections',
    'ids': 'identifiers',
    'name': 'name',
    'mf': 'manufacturer',
    'mdl': 'model',
    'sw': 'sw_version',
}

def clear_discovery_hash(hass, discovery_hash):
    """Clear entry in ALREADY_DISCOVERED list."""
    del hass.data[ALREADY_DISCOVERED][discovery_hash]

async def async_start(hass: HomeAssistantType, discovery_topic, hass_config,
                      config_entry=None) -> bool:
    """Initialize of MQTT Discovery."""
    async def async_device_message_received(topic, payload, qos):
        """Process the received message."""
        match = TOPIC_MATCHER.match(topic)

        if not match:
            return

        _prefix_topic, component, node_id, object_id = match.groups()

        if component not in SUPPORTED_COMPONENTS:
            _LOGGER.warning("Component %s is not supported", component)
            return

        if payload:
            try:
                payload = json.loads(payload)
            except ValueError:
                _LOGGER.warning("Unable to parse JSON %s: '%s'",
                                object_id, payload)
                return

        payload = dict(payload)

        for key in list(payload.keys()):
            abbreviated_key = key
            key = ABBREVIATIONS.get(key, key)
            payload[key] = payload.pop(abbreviated_key)

        if CONF_DEVICE in payload:
            device = payload[CONF_DEVICE]
            for key in list(device.keys()):
                abbreviated_key = key
                key = DEVICE_ABBREVIATIONS.get(key, key)
                device[key] = device.pop(abbreviated_key)

        base = payload.pop(TOPIC_BASE, None)
        if base:
            for key, value in payload.items():
                if isinstance(value, str) and value:
                    if value[0] == TOPIC_BASE and key.endswith('_topic'):
                        payload[key] = "{}{}".format(base, value[1:])
                    if value[-1] == TOPIC_BASE and key.endswith('_topic'):
                        payload[key] = "{}{}".format(value[:-1], base)

        # If present, the node_id will be included in the discovered object id
        discovery_id = ' '.join((node_id, object_id)) if node_id else object_id
        discovery_hash = (component, discovery_id)

        if payload:
            if CONF_PLATFORM in payload and 'schema' not in payload:
                platform = payload[CONF_PLATFORM]
                if (component in DEPRECATED_PLATFORM_TO_SCHEMA and
                        platform in DEPRECATED_PLATFORM_TO_SCHEMA[component]):
                    schema = DEPRECATED_PLATFORM_TO_SCHEMA[component][platform]
                    payload['schema'] = schema
                    _LOGGER.warning('"platform": "%s" is deprecated, '
                                    'replace with "schema":"%s"',
                                    platform, schema)
            payload[CONF_PLATFORM] = 'mqtt'

            if CONF_STATE_TOPIC not in payload:
                payload[CONF_STATE_TOPIC] = '{}/{}/{}{}/state'.format(
                    discovery_topic, component,
                    '%s/' % node_id if node_id else '', object_id)

            payload[ATTR_DISCOVERY_HASH] = discovery_hash

        if ALREADY_DISCOVERED not in hass.data:
            hass.data[ALREADY_DISCOVERED] = {}
        if discovery_hash in hass.data[ALREADY_DISCOVERED]:
            # Dispatch update
            _LOGGER.info(
                "Component has already been discovered: %s %s, sending update",
                component, discovery_id)
            async_dispatcher_send(
                hass, MQTT_DISCOVERY_UPDATED.format(discovery_hash), payload)
        elif payload:
            # Add component
            _LOGGER.info("Found new component: %s %s", component, discovery_id)
            hass.data[ALREADY_DISCOVERED][discovery_hash] = None

            if component not in CONFIG_ENTRY_COMPONENTS:
                await async_load_platform(
                    hass, component, 'mqtt', payload, hass_config)
                return

            config_entries_key = '{}.{}'.format(component, 'mqtt')
            async with hass.data[DATA_CONFIG_ENTRY_LOCK]:
                if config_entries_key not in hass.data[CONFIG_ENTRY_IS_SETUP]:
                    await hass.config_entries.async_forward_entry_setup(
                        config_entry, component)
                    hass.data[CONFIG_ENTRY_IS_SETUP].add(config_entries_key)

            async_dispatcher_send(hass, MQTT_DISCOVERY_NEW.format(
                component, 'mqtt'), payload)

    hass.data[DATA_CONFIG_ENTRY_LOCK] = asyncio.Lock()
    hass.data[CONFIG_ENTRY_IS_SETUP] = set()

    await mqtt.async_subscribe(
        hass, discovery_topic + '/#', async_device_message_received, 0)

    return True
<commit_msg>fix empty TOPIC_BASE issue (#21740)<commit_after>"""
Support for MQTT discovery.

For more details about this component, please refer to the documentation at
https://home-assistant.io/components/mqtt/#discovery
"""
import asyncio
import json
import logging
import re

from homeassistant.components import mqtt
from homeassistant.components.mqtt import ATTR_DISCOVERY_HASH, CONF_STATE_TOPIC
from homeassistant.const import CONF_DEVICE, CONF_PLATFORM
from homeassistant.helpers.discovery import async_load_platform
from homeassistant.helpers.dispatcher import async_dispatcher_send
from homeassistant.helpers.typing import HomeAssistantType

_LOGGER = logging.getLogger(__name__)

TOPIC_MATCHER = re.compile(
    r'(?P<prefix_topic>\w+)/(?P<component>\w+)/'
    r'(?:(?P<node_id>[a-zA-Z0-9_-]+)/)?(?P<object_id>[a-zA-Z0-9_-]+)/config')

SUPPORTED_COMPONENTS = [
    'binary_sensor', 'camera', 'cover', 'fan',
    'light', 'sensor', 'switch', 'lock', 'climate',
    'alarm_control_panel', 'vacuum']

CONFIG_ENTRY_COMPONENTS = [
    'binary_sensor',
    'camera',
    'cover',
    'light',
    'lock',
    'sensor',
    'switch',
    'climate',
    'alarm_control_panel',
    'fan',
    'vacuum',
]

DEPRECATED_PLATFORM_TO_SCHEMA = {
    'light': {
        'mqtt_json': 'json',
        'mqtt_template': 'template',
    }
}

ALREADY_DISCOVERED = 'mqtt_discovered_components'
DATA_CONFIG_ENTRY_LOCK = 'mqtt_config_entry_lock'
CONFIG_ENTRY_IS_SETUP = 'mqtt_config_entry_is_setup'
MQTT_DISCOVERY_UPDATED = 'mqtt_discovery_updated_{}'
MQTT_DISCOVERY_NEW = 'mqtt_discovery_new_{}_{}'

TOPIC_BASE = '~'

ABBREVIATIONS = {
    'aux_cmd_t': 'aux_command_topic',
    'aux_stat_tpl': 'aux_state_template',
    'aux_stat_t': 'aux_state_topic',
    'avty_t': 'availability_topic',
    'away_mode_cmd_t': 'away_mode_command_topic',
    'away_mode_stat_tpl': 'away_mode_state_template',
    'away_mode_stat_t': 'away_mode_state_topic',
    'bri_cmd_t': 'brightness_command_topic',
    'bri_scl': 'brightness_scale',
    'bri_stat_t': 'brightness_state_topic',
    'bri_val_tpl': 'brightness_value_template',
    'clr_temp_cmd_tpl': 'color_temp_command_template',
    'bat_lev_t': 'battery_level_topic',
    'bat_lev_tpl': 'battery_level_template',
    'chrg_t': 'charging_topic',
    'chrg_tpl': 'charging_template',
    'clr_temp_cmd_t': 'color_temp_command_topic',
    'clr_temp_stat_t': 'color_temp_state_topic',
    'clr_temp_val_tpl': 'color_temp_value_template',
    'cln_t': 'cleaning_topic',
    'cln_tpl': 'cleaning_template',
    'cmd_t': 'command_topic',
    'curr_temp_t': 'current_temperature_topic',
    'dev': 'device',
    'dev_cla': 'device_class',
    'dock_t': 'docked_topic',
    'dock_tpl': 'docked_template',
    'err_t': 'error_topic',
    'err_tpl': 'error_template',
    'fanspd_t': 'fan_speed_topic',
    'fanspd_tpl': 'fan_speed_template',
    'fanspd_lst': 'fan_speed_list',
    'fx_cmd_t': 'effect_command_topic',
    'fx_list': 'effect_list',
    'fx_stat_t': 'effect_state_topic',
    'fx_val_tpl': 'effect_value_template',
    'exp_aft': 'expire_after',
    'fan_mode_cmd_t': 'fan_mode_command_topic',
    'fan_mode_stat_tpl': 'fan_mode_state_template',
    'fan_mode_stat_t': 'fan_mode_state_topic',
    'frc_upd': 'force_update',
    'hold_cmd_t': 'hold_command_topic',
    'hold_stat_tpl': 'hold_state_template',
    'hold_stat_t': 'hold_state_topic',
    'ic': 'icon',
    'init': 'initial',
    'json_attr': 'json_attributes',
    'json_attr_t': 'json_attributes_topic',
    'max_temp': 'max_temp',
    'min_temp': 'min_temp',
    'mode_cmd_t': 'mode_command_topic',
    'mode_stat_tpl': 'mode_state_template',
    'mode_stat_t': 'mode_state_topic',
    'name': 'name',
    'on_cmd_type': 'on_command_type',
    'opt': 'optimistic',
    'osc_cmd_t': 'oscillation_command_topic',
    'osc_stat_t': 'oscillation_state_topic',
    'osc_val_tpl': 'oscillation_value_template',
    'pl_arm_away': 'payload_arm_away',
    'pl_arm_home': 'payload_arm_home',
    'pl_avail': 'payload_available',
    'pl_cls': 'payload_close',
    'pl_disarm': 'payload_disarm',
    'pl_hi_spd': 'payload_high_speed',
    'pl_lock': 'payload_lock',
    'pl_lo_spd': 'payload_low_speed',
    'pl_med_spd': 'payload_medium_speed',
    'pl_not_avail': 'payload_not_available',
    'pl_off': 'payload_off',
    'pl_on': 'payload_on',
    'pl_open': 'payload_open',
    'pl_osc_off': 'payload_oscillation_off',
    'pl_osc_on': 'payload_oscillation_on',
    'pl_stop': 'payload_stop',
    'pl_unlk': 'payload_unlock',
    'pow_cmd_t': 'power_command_topic',
    'ret': 'retain',
    'rgb_cmd_tpl': 'rgb_command_template',
    'rgb_cmd_t': 'rgb_command_topic',
    'rgb_stat_t': 'rgb_state_topic',
    'rgb_val_tpl': 'rgb_value_template',
    'send_cmd_t': 'send_command_topic',
    'send_if_off': 'send_if_off',
    'set_pos_tpl': 'set_position_template',
    'set_pos_t': 'set_position_topic',
    'spd_cmd_t': 'speed_command_topic',
    'spd_stat_t': 'speed_state_topic',
    'spd_val_tpl': 'speed_value_template',
    'spds': 'speeds',
    'stat_clsd': 'state_closed',
    'stat_off': 'state_off',
    'stat_on': 'state_on',
    'stat_open': 'state_open',
    'stat_t': 'state_topic',
    'stat_val_tpl': 'state_value_template',
    'sup_feat': 'supported_features',
    'swing_mode_cmd_t': 'swing_mode_command_topic',
    'swing_mode_stat_tpl': 'swing_mode_state_template',
    'swing_mode_stat_t': 'swing_mode_state_topic',
    'temp_cmd_t': 'temperature_command_topic',
    'temp_stat_tpl': 'temperature_state_template',
    'temp_stat_t': 'temperature_state_topic',
    'tilt_clsd_val': 'tilt_closed_value',
    'tilt_cmd_t': 'tilt_command_topic',
    'tilt_inv_stat': 'tilt_invert_state',
    'tilt_max': 'tilt_max',
    'tilt_min': 'tilt_min',
    'tilt_opnd_val': 'tilt_opened_value',
    'tilt_status_opt': 'tilt_status_optimistic',
    'tilt_status_t': 'tilt_status_topic',
    't': 'topic',
    'uniq_id': 'unique_id',
    'unit_of_meas': 'unit_of_measurement',
    'val_tpl': 'value_template',
    'whit_val_cmd_t': 'white_value_command_topic',
    'whit_val_scl': 'white_value_scale',
    'whit_val_stat_t': 'white_value_state_topic',
    'whit_val_tpl': 'white_value_template',
    'xy_cmd_t': 'xy_command_topic',
    'xy_stat_t': 'xy_state_topic',
    'xy_val_tpl': 'xy_value_template',
}

DEVICE_ABBREVIATIONS = {
    'cns': 'connections',
    'ids': 'identifiers',
    'name': 'name',
    'mf': 'manufacturer',
    'mdl': 'model',
    'sw': 'sw_version',
}

def clear_discovery_hash(hass, discovery_hash):
    """Clear entry in ALREADY_DISCOVERED list."""
    del hass.data[ALREADY_DISCOVERED][discovery_hash]

async def async_start(hass: HomeAssistantType, discovery_topic, hass_config,
                      config_entry=None) -> bool:
    """Initialize of MQTT Discovery."""
    async def async_device_message_received(topic, payload, qos):
        """Process the received message."""
        match = TOPIC_MATCHER.match(topic)

        if not match:
            return

        _prefix_topic, component, node_id, object_id = match.groups()

        if component not in SUPPORTED_COMPONENTS:
            _LOGGER.warning("Component %s is not supported", component)
            return

        if payload:
            try:
                payload = json.loads(payload)
            except ValueError:
                _LOGGER.warning("Unable to parse JSON %s: '%s'",
                                object_id, payload)
                return

        payload = dict(payload)

        for key in list(payload.keys()):
            abbreviated_key = key
            key = ABBREVIATIONS.get(key, key)
            payload[key] = payload.pop(abbreviated_key)

        if CONF_DEVICE in payload:
            device = payload[CONF_DEVICE]
            for key in list(device.keys()):
                abbreviated_key = key
                key = DEVICE_ABBREVIATIONS.get(key, key)
                device[key] = device.pop(abbreviated_key)

        if TOPIC_BASE in payload:
            base = payload.pop(TOPIC_BASE)
            for key, value in payload.items():
                if isinstance(value, str) and value:
                    if value[0] == TOPIC_BASE and key.endswith('_topic'):
                        payload[key] = "{}{}".format(base, value[1:])
                    if value[-1] == TOPIC_BASE and key.endswith('_topic'):
                        payload[key] = "{}{}".format(value[:-1], base)

        # If present, the node_id will be included in the discovered object id
        discovery_id = ' '.join((node_id, object_id)) if node_id else object_id
        discovery_hash = (component, discovery_id)

        if payload:
            if CONF_PLATFORM in payload and 'schema' not in payload:
                platform = payload[CONF_PLATFORM]
                if (component in DEPRECATED_PLATFORM_TO_SCHEMA and
                        platform in DEPRECATED_PLATFORM_TO_SCHEMA[component]):
                    schema = DEPRECATED_PLATFORM_TO_SCHEMA[component][platform]
                    payload['schema'] = schema
                    _LOGGER.warning('"platform": "%s" is deprecated, '
                                    'replace with "schema":"%s"',
                                    platform, schema)
            payload[CONF_PLATFORM] = 'mqtt'

            if CONF_STATE_TOPIC not in payload:
                payload[CONF_STATE_TOPIC] = '{}/{}/{}{}/state'.format(
                    discovery_topic, component,
                    '%s/' % node_id if node_id else '', object_id)

            payload[ATTR_DISCOVERY_HASH] = discovery_hash

        if ALREADY_DISCOVERED not in hass.data:
            hass.data[ALREADY_DISCOVERED] = {}
        if discovery_hash in hass.data[ALREADY_DISCOVERED]:
            # Dispatch update
            _LOGGER.info(
                "Component has already been discovered: %s %s, sending update",
                component, discovery_id)
            async_dispatcher_send(
                hass, MQTT_DISCOVERY_UPDATED.format(discovery_hash), payload)
        elif payload:
            # Add component
            _LOGGER.info("Found new component: %s %s", component, discovery_id)
            hass.data[ALREADY_DISCOVERED][discovery_hash] = None

            if component not in CONFIG_ENTRY_COMPONENTS:
                await async_load_platform(
                    hass, component, 'mqtt', payload, hass_config)
                return

            config_entries_key = '{}.{}'.format(component, 'mqtt')
            async with hass.data[DATA_CONFIG_ENTRY_LOCK]:
                if config_entries_key not in hass.data[CONFIG_ENTRY_IS_SETUP]:
                    await hass.config_entries.async_forward_entry_setup(
                        config_entry, component)
                    hass.data[CONFIG_ENTRY_IS_SETUP].add(config_entries_key)

            async_dispatcher_send(hass, MQTT_DISCOVERY_NEW.format(
                component, 'mqtt'), payload)

    hass.data[DATA_CONFIG_ENTRY_LOCK] = asyncio.Lock()
    hass.data[CONFIG_ENTRY_IS_SETUP] = set()

    await mqtt.async_subscribe(
        hass, discovery_topic + '/#', async_device_message_received, 0)

    return True
<|endoftext|>